@xagents_in/unmask 0.0.0-stage → 0.1.1

package/dist/core/online.mjs

@@ -0,0 +1,310 @@
+import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
+import path from "node:path";
+import {
+  getRegistryRelease,
+  fetchBlob,
+  parsePackageSpec,
+  AcquisitionError,
+} from "./acquire.mjs";
+import { resolveSource } from "./resolve.mjs";
+import {
+  analyzeLocal,
+  LocalInputError,
+  effectiveLimits,
+  preserveInputBytes,
+} from "./pipeline.mjs";
+import { createReport, validateReport } from "./report.mjs";
+import { addManifestObservations } from "./manifests.mjs";
+
+function integrity(raw, actual) {
+  const values = [];
+  let malformed = raw !== null && raw !== undefined && typeof raw !== "string";
+  const recognized = { sha1: 20, sha256: 32, sha384: 48, sha512: 64 };
+  const unsupported = new Set();
+  for (const token of typeof raw === "string"
+    ? raw.trim().split(/\s+/).filter(Boolean)
+    : []) {
+    const expression = token.split("?")[0];
+    const dash = expression.indexOf("-");
+    const algorithm = dash === -1 ? expression : expression.slice(0, dash);
+    const encoded = dash === -1 ? "" : expression.slice(dash + 1);
+    if (!Object.hasOwn(recognized, algorithm)) {
+      unsupported.add(algorithm);
+      continue;
+    }
+    const normalized = encoded.replace(/-/g, "+").replace(/_/g, "/");
+    const bytes = Buffer.from(normalized, "base64");
+    const canonical = bytes.toString("base64");
+    if (
+      !/^[A-Za-z0-9+/_-]+={0,2}$/.test(encoded) ||
+      bytes.length !== recognized[algorithm] ||
+      canonical.replace(/=+$/, "") !== normalized.replace(/=+$/, "") ||
+      (encoded.includes("=") && canonical !== normalized)
+    ) {
+      malformed = true;
+      continue;
+    }
+    if (algorithm === "sha256" || algorithm === "sha512")
+      values.push({ algorithm, digest: bytes.toString("hex") });
+    else unsupported.add(algorithm);
+  }
+  const algorithm = values.some((value) => value.algorithm === "sha512")
+    ? "sha512"
+    : "sha256";
+  const candidates = values
+    .filter((value) => value.algorithm === algorithm)
+    .sort((a, b) => (a.digest < b.digest ? -1 : a.digest > b.digest ? 1 : 0));
+  const selected =
+    candidates.find((value) => value.digest === actual[algorithm]) ||
+    candidates[0];
+  const expected = selected
+    ? {
+        ...selected,
+        status: selected.digest === actual[algorithm] ? "match" : "mismatch",
+        ...(unsupported.size
+          ? { unsupportedAlgorithms: [...unsupported].sort() }
+          : {}),
+      }
+    : {
+        status: "unavailable",
+        reason: unsupported.size
+          ? "unsupported-algorithm"
+          : "missing-or-invalid",
+      };
+  return {
+    expected,
+    failures: malformed
+      ? [
+          {
+            code: "invalid-registry-integrity",
+            message: "Recognized registry integrity metadata is malformed",
+          },
+        ]
+      : [],
+  };
+}
+function rebuild(report, updates) {
+  const next = createReport({
+    inputs: updates.inputs || report.inputs,
+    source: updates.source || report.source,
+    comparison: {
+      files: report.files,
+      coverage: report.coverage,
+      evidence: report.evidence,
+    },
+    manifests: updates.manifests || report.manifests,
+    failures: updates.failures || report.failures,
+    run: updates.run || report.run,
+    analyzerVersion: report.analyzerVersion,
+    limitsVersion: report.limitsVersion,
+  });
+  next.cues = report.cues;
+  next.annotations = report.annotations;
+  next.model = report.model;
+  return validateReport(next);
+}
+export async function addConfiguredModel(report, model) {
+  if (model === undefined) return report;
+  if (typeof model !== "function")
+    return validateReport({
+      ...report,
+      model: { status: "unavailable", reason: "not-configured" },
+    });
+  try {
+    const section = await model(structuredClone(report));
+    return validateReport({ ...report, model: section });
+  } catch {
+    return validateReport({
+      ...report,
+      model: { status: "unavailable", reason: "model-request-failed" },
+    });
+  }
+}
+
+export async function analyzeRegistry(
+  spec,
+  {
+    workDir,
+    fetchImpl,
+    signal,
+    limits: overrides,
+    model,
+    persistCache = true,
+  } = {},
+) {
+  let identity;
+  try {
+    identity = parsePackageSpec(spec);
+  } catch (error) {
+    if (error instanceof AcquisitionError)
+      throw new LocalInputError(error.code, error.message);
+    throw error;
+  }
+  if (typeof workDir !== "string" || !workDir)
+    throw new LocalInputError(
+      "work-dir-required",
+      "Registry analysis requires an explicit private work directory",
+    );
+  if (typeof persistCache !== "boolean")
+    throw new LocalInputError(
+      "invalid-cache-policy",
+      "persistCache must be an explicit boolean",
+    );
+  const limits = effectiveLimits(overrides);
+  await mkdir(workDir, { recursive: true, mode: 0o700 });
+  const jobDir = await mkdtemp(path.join(workDir, "job-"));
+  const observations = [];
+  try {
+    let release, artifact;
+    try {
+      release = await getRegistryRelease(identity, { fetchImpl, signal });
+      observations.push(release.observation);
+      await writeFile(
+        path.join(jobDir, "registry.json"),
+        release.metadataBytes,
+        { flag: "wx", mode: 0o600 },
+      );
+      if (persistCache)
+        await preserveInputBytes(
+          workDir,
+          { digest: release.metadataDigest },
+          release.metadataBytes,
+          observations,
+        );
+      artifact = await fetchBlob(release.tarballUrl, {
+        fetchImpl,
+        signal,
+        maxBytes: limits.compressedBytes,
+      });
+      observations.push(artifact.observation);
+    } catch (error) {
+      if (error instanceof AcquisitionError)
+        throw new LocalInputError(error.code, error.message);
+      throw error;
+    }
+    const tarballPath = path.join(jobDir, "artifact.tgz");
+    await writeFile(tarballPath, artifact.bytes, { flag: "wx", mode: 0o600 });
+    const resolution = resolveSource({
+      claim: identity,
+      registryDoc: release.metadata,
+    });
+    const candidate = resolution.candidate;
+    const sourceOptions = {};
+    const failures = [];
+    let sourceFetchFailed = false;
+    if (candidate) {
+      try {
+        const source = await fetchBlob(candidate.archiveUrl, {
+          fetchImpl,
+          signal,
+          maxBytes: limits.compressedBytes,
+        });
+        observations.push(source.observation);
+        sourceOptions.sourceArchivePath = path.join(jobDir, "source.tgz");
+        sourceOptions.commit = candidate.commit;
+        sourceOptions.sourceRoot = candidate.sourceRoot;
+        if (
+          typeof release.metadata.repository === "object" &&
+          release.metadata.repository !== null &&
+          Object.hasOwn(release.metadata.repository, "directory")
+        )
+          sourceOptions.sourceDir = candidate.directory;
+        await writeFile(sourceOptions.sourceArchivePath, source.bytes, {
+          flag: "wx",
+          mode: 0o600,
+        });
+      } catch (error) {
+        if (!(error instanceof AcquisitionError)) throw error;
+        if (error.observation) observations.push(error.observation);
+        sourceFetchFailed = true;
+        failures.push({
+          code: "source-acquisition-failed",
+          message: "Selected publisher source archive could not be acquired",
+          reason: error.code,
+        });
+      }
+    }
+    const report = await analyzeLocal({
+      tarballPath,
+      ...sourceOptions,
+      claim: identity,
+      limits: overrides,
+      cacheDir: persistCache ? workDir : undefined,
+    });
+    const checked = integrity(
+      release.expectedIntegrity,
+      report.inputs.artifact,
+    );
+    if (
+      release.metadata.dist?.integrity !== undefined &&
+      typeof release.metadata.dist.integrity !== "string"
+    )
+      checked.failures.push({
+        code: "invalid-registry-integrity",
+        message: "Registry integrity must be a string",
+      });
+    const source = candidate
+      ? {
+          ...report.source,
+          method: candidate.method,
+          repository: candidate.repository,
+          commit: candidate.commit,
+          alternatives: resolution.alternatives,
+          ...(sourceFetchFailed
+            ? {
+                status: "unavailable",
+                complete: false,
+                reason: "source-acquisition-failed",
+              }
+            : {}),
+        }
+      : {
+          ...report.source,
+          method: resolution.method,
+          status: resolution.status,
+          alternatives: resolution.alternatives,
+          reason: resolution.reason || "no-source-candidate",
+        };
+    const inputs = {
+      ...report.inputs,
+      artifact: {
+        ...report.inputs.artifact,
+        kind: "registry",
+        expectedIntegrity: checked.expected,
+        provenance: { status: "not-assessed" },
+      },
+      registryDocument: { ...release.metadataDigest, kind: "registry" },
+      limits: {
+        ...report.inputs.limits,
+        acquisition: {
+          metadataBytes: 1048576,
+          timeoutMs: 30000,
+          redirects: "forbidden",
+        },
+      },
+      ...(report.inputs.sourceArchive
+        ? { sourceArchive: { ...report.inputs.sourceArchive, kind: "remote" } }
+        : {}),
+    };
+    const next = rebuild(report, {
+      inputs,
+      source,
+      manifests: {
+        ...report.manifests,
+        registry: {
+          status: "usable",
+          value: release.metadata,
+          evidenceIds: [],
+        },
+      },
+      failures: [...report.failures, ...failures, ...checked.failures],
+      run: {
+        ...report.run,
+        observations: [...report.run.observations, ...observations],
+      },
+    });
+    return await addConfiguredModel(addManifestObservations(next), model);
+  } finally {
+    await rm(jobDir, { recursive: true, force: true });
+  }
+}

package/dist/core/pipeline.mjs

@@ -0,0 +1,345 @@
+import { constants } from "node:fs";
+import { open, mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { readArchive, DEFAULT_LIMITS } from "./archive.mjs";
+import { selectSourcePackage } from "./source.mjs";
+import { compareInventories } from "./compare.mjs";
+import { createReport } from "./report.mjs";
+import { inspectJavaScript, addSyntaxInspection } from "./syntax.mjs";
+import { addManifestObservations } from "./manifests.mjs";
+
+export class LocalInputError extends Error {
+  constructor(code, message) {
+    super(message);
+    this.name = "LocalInputError";
+    this.code = code;
+    this.exitCode = 2;
+  }
+}
+function limitsFor(overrides = {}) {
+  if (!overrides || typeof overrides !== "object" || Array.isArray(overrides))
+    throw new LocalInputError("invalid-limits", "Limits must be an object");
+  const defaults = { ...DEFAULT_LIMITS, retentionBudgetBytes: 134217728 };
+  for (const [key, value] of Object.entries(overrides))
+    if (
+      !(key in defaults) ||
+      !Number.isSafeInteger(value) ||
+      value < 0 ||
+      value > defaults[key]
+    )
+      throw new LocalInputError(
+        "invalid-limits",
+        "Local limits may only lower finite defaults",
+      );
+  return { ...defaults, ...overrides };
+}
+function safeDirectory(value, allowEmpty = false) {
+  return (
+    typeof value === "string" &&
+    ((allowEmpty && value === "") ||
+      (value.length > 0 &&
+        !/^[a-z]:/i.test(value) &&
+        !/[\0\\]/.test(value) &&
+        !value.startsWith("/") &&
+        value
+          .split("/")
+          .every((part) => part && part !== "." && part !== "..")))
+  );
+}
+function validateOptions(options) {
+  if (typeof options.tarballPath !== "string" || !options.tarballPath)
+    throw new LocalInputError(
+      "artifact-required",
+      "A local artifact path is required",
+    );
+  if (options.sourceArchivePath !== undefined) {
+    if (
+      typeof options.sourceArchivePath !== "string" ||
+      !options.sourceArchivePath ||
+      !/^[a-f0-9]{40}$/.test(options.commit || "") ||
+      !safeDirectory(options.sourceRoot)
+    )
+      throw new LocalInputError(
+        "invalid-source-claim",
+        "Selected source requires a local archive, full lowercase commit SHA and explicit safe root",
+      );
+  } else if (
+    options.commit !== undefined ||
+    options.sourceRoot !== undefined ||
+    options.sourceDir !== undefined
+  )
+    throw new LocalInputError(
+      "source-required",
+      "Source options require an explicit local source archive",
+    );
+  if (
+    options.sourceDir !== undefined &&
+    !safeDirectory(options.sourceDir, true)
+  )
+    throw new LocalInputError(
+      "invalid-source-directory",
+      "Source directory must be a safe relative path",
+    );
+  if (
+    options.claim !== undefined &&
+    (!options.claim ||
+      typeof options.claim !== "object" ||
+      typeof options.claim.name !== "string" ||
+      typeof options.claim.version !== "string")
+  )
+    throw new LocalInputError(
+      "invalid-claim",
+      "Identity claim requires an exact name and version",
+    );
+  if (
+    options.cacheDir !== undefined &&
+    (typeof options.cacheDir !== "string" || !options.cacheDir)
+  )
+    throw new LocalInputError(
+      "invalid-cache",
+      "Cache directory must be an explicit path",
+    );
+}
+async function boundedRead(filename, cap, side) {
+  let handle;
+  try {
+    handle = await open(filename, constants.O_RDONLY | constants.O_NONBLOCK);
+    const before = await handle.stat({ bigint: true });
+    if (!before.isFile())
+      throw new LocalInputError(
+        `${side}-read-failed`,
+        "Input must be a regular local file",
+      );
+    if (before.size > BigInt(cap))
+      throw new LocalInputError(
+        `${side}-compressed-cap`,
+        "Compressed input exceeds the configured byte cap",
+      );
+    const bytes = Buffer.allocUnsafe(Number(before.size));
+    let offset = 0;
+    while (offset < bytes.length) {
+      const { bytesRead } = await handle.read(
+        bytes,
+        offset,
+        Math.min(bytes.length - offset, 65536),
+        offset,
+      );
+      if (bytesRead === 0)
+        throw new LocalInputError(
+          `${side}-read-changed`,
+          "Local file changed during bounded read",
+        );
+      offset += bytesRead;
+    }
+    const extra = await handle.read(Buffer.alloc(1), 0, 1, offset);
+    const after = await handle.stat({ bigint: true });
+    if (
+      extra.bytesRead ||
+      before.size !== after.size ||
+      before.mtimeNs !== after.mtimeNs ||
+      before.ctimeNs !== after.ctimeNs
+    )
+      throw new LocalInputError(
+        `${side}-read-changed`,
+        "Local file changed during bounded read",
+      );
+    return bytes;
+  } catch (error) {
+    if (error instanceof LocalInputError) throw error;
+    if (typeof error.code === "string" && /^E[A-Z]+$/.test(error.code))
+      throw new LocalInputError(
+        `${side}-read-failed`,
+        "Unable to read the selected local input",
+      );
+    throw error;
+  } finally {
+    if (handle) await handle.close();
+  }
+}
+async function cacheBytes(cacheDir, inventory, bytes, observations) {
+  if (cacheDir === undefined) return;
+  const directory = path.join(cacheDir, "blobs", "sha256");
+  const destination = path.join(directory, inventory.digest.sha256);
+  try {
+    await mkdir(directory, { recursive: true, mode: 0o700 });
+    try {
+      await writeFile(destination, bytes, { flag: "wx", mode: 0o600 });
+    } catch (error) {
+      if (error.code !== "EEXIST") throw error;
+      const existing = await boundedRead(destination, bytes.length, "cache");
+      if (!existing.equals(bytes)) throw new Error("Cache bytes differ");
+    }
+  } catch {
+    observations.push({
+      code: "cache-write-failed",
+      message:
+        "Original input was analyzed, but its private cache copy could not be verified",
+    });
+  }
+}
+function manifestFor(inventory, comparison, side) {
+  const entry = inventory?.entries.find(
+    (item) => item.type === "file" && item.path === "package.json",
+  );
+  const evidenceIds = comparison.evidence
+    .filter((e) => e.side === side && e.path === "package.json")
+    .map((e) => e.id);
+  if (!entry || !Buffer.isBuffer(entry.content))
+    return {
+      status: "unavailable",
+      reason: entry?.inspection || "missing-or-nonregular",
+    };
+  try {
+    const value = JSON.parse(
+      new TextDecoder("utf-8", { fatal: true }).decode(entry.content),
+    );
+    if (!value || typeof value !== "object" || Array.isArray(value))
+      return { status: "invalid", reason: "not-an-object" };
+    return { status: "usable", value, evidenceIds };
+  } catch {
+    return { status: "invalid", reason: "invalid-json-or-encoding" };
+  }
+}
+function artifactName(inventory) {
+  const entry = inventory.entries.find(
+    (item) => item.type === "file" && item.path === "package.json",
+  );
+  if (!Buffer.isBuffer(entry?.content)) return undefined;
+  try {
+    return JSON.parse(
+      new TextDecoder("utf-8", { fatal: true }).decode(entry.content),
+    )?.name;
+  } catch {
+    return undefined;
+  }
+}
+
+export async function analyzeLocal(options) {
+  validateOptions(options);
+  const limits = limitsFor(options.limits);
+  const { retentionBudgetBytes, ...archiveLimits } = limits;
+  const retentionBudget = { limitBytes: retentionBudgetBytes, usedBytes: 0 };
+  const observations = [];
+  const failures = [];
+  const artifactBytes = await boundedRead(
+    options.tarballPath,
+    limits.compressedBytes,
+    "artifact",
+  );
+  const artifact = await readArchive(artifactBytes, {
+    kind: "artifact",
+    limits: archiveLimits,
+    retentionBudget,
+  });
+  failures.push(...artifact.failures.map((f) => ({ ...f, side: "artifact" })));
+  await cacheBytes(options.cacheDir, artifact, artifactBytes, observations);
+  let source = {
+    status: "unresolved",
+    method: "unresolved",
+    complete: false,
+    alternatives: [],
+  };
+  let sourceArchive = null;
+  let selected = null;
+  if (options.sourceArchivePath !== undefined) {
+    source = {
+      status: "unavailable",
+      method: "user-full-SHA",
+      commit: options.commit,
+      complete: false,
+      alternatives: [],
+    };
+    try {
+      const sourceBytes = await boundedRead(
+        options.sourceArchivePath,
+        limits.compressedBytes,
+        "source",
+      );
+      sourceArchive = await readArchive(sourceBytes, {
+        kind: "source",
+        sourceRoot: options.sourceRoot,
+        limits: archiveLimits,
+        retentionBudget,
+      });
+      await cacheBytes(
+        options.cacheDir,
+        sourceArchive,
+        sourceBytes,
+        observations,
+      );
+      failures.push(
+        ...sourceArchive.failures.map((f) => ({ ...f, side: "source" })),
+      );
+      const selection = selectSourcePackage(sourceArchive, {
+        ...(options.sourceDir === undefined
+          ? {}
+          : { directory: options.sourceDir }),
+        packageName: artifactName(artifact),
+      });
+      source.status = selection.status;
+      source.reason =
+        selection.reason ||
+        (sourceArchive.complete
+          ? "explicit-local-source"
+          : "incomplete-source-inventory");
+      if (selection.directory !== null) source.directory = selection.directory;
+      if (selection.status === "selected") {
+        selected = { ...sourceArchive, entries: selection.entries };
+        source.complete = sourceArchive.complete;
+      } else
+        failures.push({
+          code: "source-selection-unresolved",
+          message:
+            "Selected source package could not be resolved unambiguously",
+          reason: selection.reason,
+        });
+    } catch (error) {
+      if (!(error instanceof LocalInputError)) throw error;
+      failures.push({
+        code: error.code,
+        message: error.message,
+        side: "source",
+      });
+    }
+  }
+  const comparison = compareInventories(artifact, selected);
+  const manifests = {
+    artifact: manifestFor(artifact, comparison, "artifact"),
+    source: manifestFor(selected, comparison, "source"),
+    registry: { status: "unavailable", reason: "offline" },
+  };
+  for (const side of ["artifact", "source"])
+    if (manifests[side].status === "invalid")
+      failures.push({
+        code: `${side}-manifest-invalid`,
+        message: "Package manifest is not a valid UTF-8 JSON object",
+        side,
+      });
+  const inputs = {
+    artifact: {
+      ...artifact.digest,
+      kind: "local",
+      expectedIntegrity: { status: "unavailable" },
+    },
+    limits,
+    ...(sourceArchive
+      ? { sourceArchive: { ...sourceArchive.digest, kind: "local" } }
+      : {}),
+    ...(options.claim ? { claim: options.claim } : {}),
+  };
+  const report = createReport({
+    inputs,
+    source,
+    comparison,
+    manifests,
+    failures,
+    run: { surface: "cli", observations },
+  });
+  const inspected = addSyntaxInspection(
+    report,
+    inspectJavaScript(artifact, selected),
+  );
+  return addManifestObservations(inspected, { artifact, source: selected });
+}
+
+export { limitsFor as effectiveLimits, cacheBytes as preserveInputBytes };