@xagents_in/unmask 0.0.0-stage → 0.1.0

package/dist/model/config.mjs

@@ -0,0 +1,90 @@
+export async function modelClientFromEnv(env = process.env) {
+  if (env.UNMASK_MODEL === undefined || env.UNMASK_MODEL === "off")
+    return undefined;
+  if (
+    env.UNMASK_MODEL !== "on" ||
+    (env.UNMASK_MODEL_QUOTA_VERIFIED !== undefined &&
+      !["true", "false"].includes(env.UNMASK_MODEL_QUOTA_VERIFIED))
+  )
+    throw new TypeError("Invalid model configuration");
+  const preferred = env.GEMINI_API_KEY;
+  const alias = env.UNMASK_MODEL_KEY;
+  if (preferred !== undefined && alias !== undefined && preferred !== alias)
+    throw new TypeError("Conflicting model key configuration");
+  const apiKey = preferred ?? alias;
+  if (env.UNMASK_MODEL_QUOTA_VERIFIED === "false" && apiKey === undefined)
+    return undefined;
+  if (
+    typeof apiKey !== "string" ||
+    apiKey.length < 1 ||
+    apiKey.length > 4096 ||
+    /[\s\x00-\x1f\x7f]/u.test(apiKey)
+  )
+    throw new TypeError("Invalid model key configuration");
+  const quota = (name, fallback) => {
+    const value = env[name] ?? fallback;
+    if (
+      typeof value !== "string" ||
+      !/^[1-9][0-9]*$/.test(value) ||
+      !Number.isSafeInteger(Number(value))
+    )
+      throw new TypeError("Invalid model quota configuration");
+    return Number(value);
+  };
+  const provider = env.UNMASK_MODEL_PROVIDER ?? "google";
+  if (!["google", "gemini"].includes(provider))
+    throw new TypeError("Invalid model provider configuration");
+  const legacy =
+    env.UNMASK_MODEL_REQUESTS_PER_MINUTE !== undefined ||
+    env.UNMASK_MODEL_DAILY_REQUESTS !== undefined;
+  if (
+    legacy &&
+    (env.UNMASK_MODEL_QUOTA_VERIFIED !== "true" ||
+      env.UNMASK_MODEL_GENERATIONS_PER_MINUTE !== undefined ||
+      env.UNMASK_MODEL_GENERATIONS_PER_DAY !== undefined)
+  )
+    throw new TypeError("Conflicting model quota configuration");
+  const options = {
+    apiKey,
+    provider,
+    quotaVerified: env.UNMASK_MODEL_QUOTA_VERIFIED === "true",
+    ...(legacy
+      ? {
+          requestsPerMinute: quota("UNMASK_MODEL_REQUESTS_PER_MINUTE"),
+          inputTokensPerMinute: quota("UNMASK_MODEL_INPUT_TOKENS_PER_MINUTE"),
+          dailyRequests: quota("UNMASK_MODEL_DAILY_REQUESTS"),
+        }
+      : {
+          localLimits: {
+            generationsPerMinute: quota(
+              "UNMASK_MODEL_GENERATIONS_PER_MINUTE",
+              "3",
+            ),
+            inputTokensPerMinute: quota(
+              "UNMASK_MODEL_INPUT_TOKENS_PER_MINUTE",
+              "12000",
+            ),
+            generationsPerDay: quota("UNMASK_MODEL_GENERATIONS_PER_DAY", "100"),
+          },
+        }),
+  };
+  const { createGemmaClient, DEFAULT_MODEL_ROUTE } =
+    await import("./client.mjs");
+  if (env.UNMASK_MODEL_ROUTE !== undefined) {
+    try {
+      if (
+        typeof env.UNMASK_MODEL_ROUTE !== "string" ||
+        env.UNMASK_MODEL_ROUTE.length > 2048
+      )
+        throw new Error();
+      options.route = JSON.parse(env.UNMASK_MODEL_ROUTE);
+    } catch {
+      throw new TypeError("Invalid model route configuration");
+    }
+    if (env.UNMASK_MODEL_ID !== undefined)
+      options.modelId = env.UNMASK_MODEL_ID;
+  } else if (env.UNMASK_MODEL_ID !== undefined)
+    options.modelId = env.UNMASK_MODEL_ID;
+  else options.route = DEFAULT_MODEL_ROUTE;
+  return createGemmaClient(options);
+}

package/dist/server/app.mjs

@@ -0,0 +1,299 @@
+import { createServer } from "node:http";
+import { createQueue, resolveLimits, systemClock } from "./queue.mjs";
+import { createReportStore } from "./store.mjs";
+import { renderReport } from "../viewer/render.mjs";
+import {
+  renderHome,
+  renderStatus,
+  renderError,
+  renderCompleted,
+} from "../viewer/shell.mjs";
+
+export function parseNameAtExactVersion(value) {
+  if (typeof value !== "string" || value.length > 300 || value !== value.trim())
+    return null;
+  const match =
+    /^(?<name>(?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*)@(?<version>(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?)$/.exec(
+      value,
+    );
+  if (
+    !match ||
+    match.groups.name.length > 214 ||
+    ["node_modules", "favicon.ico"].includes(match.groups.name)
+  )
+    return null;
+  const prerelease = match.groups.version
+    .split("+")[0]
+    .split("-")
+    .slice(1)
+    .join("-");
+  if (prerelease.split(".").some((part) => /^0\d+$/.test(part))) return null;
+  if (
+    match.groups.version
+      .split(/[.+-]/)
+      .slice(0, 3)
+      .some((part) => !Number.isSafeInteger(Number(part)))
+  )
+    return null;
+  return `${match.groups.name}@${match.groups.version}`;
+}
+
+function bodyError(status) {
+  return Object.assign(new Error("Request body rejected"), { status });
+}
+function readBody(req, limits) {
+  return new Promise((resolve, reject) => {
+    let size = 0,
+      chunks = [],
+      done = false;
+    const timer = setTimeout(
+      () => finish(bodyError(408)),
+      limits.bodyTimeoutMs,
+    );
+    timer.unref();
+    function finish(error) {
+      if (done) return;
+      done = true;
+      clearTimeout(timer);
+      req.removeListener("data", data);
+      req.removeListener("end", end);
+      req.removeListener("error", fail);
+      req.removeListener("aborted", fail);
+      if (error) {
+        chunks = [];
+        req.on("error", () => {});
+        req.resume();
+        reject(error);
+      } else resolve(Buffer.concat(chunks, size).toString("utf8"));
+    }
+    const data = (chunk) => {
+      size += chunk.length;
+      if (size > limits.maxBodyBytes) finish(bodyError(413));
+      else chunks.push(chunk);
+    };
+    const end = () => finish();
+    const fail = () => finish(bodyError(400));
+    if (Number(req.headers["content-length"]) > limits.maxBodyBytes) {
+      finish(bodyError(413));
+      return;
+    }
+    req.on("data", data);
+    req.on("end", end);
+    req.on("error", fail);
+    req.on("aborted", fail);
+  });
+}
+
+export function createCacheKeyPrefix(versionInfo, limits) {
+  return JSON.stringify([
+    versionInfo.version,
+    versionInfo.commit,
+    versionInfo.tarballSha256,
+    versionInfo.analyzerVersion,
+    versionInfo.policyVersion,
+    versionInfo.limitsVersion,
+    versionInfo.modelConfigDigest,
+    limits,
+  ]);
+}
+
+export function createApp({
+  runAnalysis,
+  versionInfo,
+  limits: overrides,
+  clock = systemClock,
+} = {}) {
+  if (typeof runAnalysis !== "function")
+    throw new TypeError("runAnalysis is required");
+  if (
+    !versionInfo ||
+    ["version", "commit", "tarballSha256"].some(
+      (key) => typeof versionInfo[key] !== "string",
+    )
+  )
+    throw new TypeError(
+      "versionInfo requires version, commit and tarballSha256 strings",
+    );
+  const limits = resolveLimits(overrides);
+  const version = Object.fromEntries(
+    ["version", "commit", "tarballSha256"].map((key) => [
+      key,
+      versionInfo[key],
+    ]),
+  );
+  const store = createReportStore({
+    maxEntries: limits.maxReports,
+    maxBytes: limits.maxReportBytes,
+    ttlMs: limits.reportTtlMs,
+    clock,
+  });
+  const keyPrefix = createCacheKeyPrefix(versionInfo, limits);
+  const queue = createQueue({ runAnalysis, store, limits, clock, keyPrefix });
+  const server = createServer(
+    { maxHeaderSize: 8192, headersTimeout: 10000, requestTimeout: 15000 },
+    (req, res) => {
+      function send(
+        status,
+        body,
+        type = "text/html; charset=utf-8",
+        headers = {},
+      ) {
+        if (res.destroyed || res.writableEnded) return;
+        res.writeHead(status, {
+          "content-type": type,
+          "x-content-type-options": "nosniff",
+          "referrer-policy": "no-referrer",
+          "cache-control": "no-store",
+          "x-frame-options": "DENY",
+          ...headers,
+        });
+        res.end(body);
+      }
+      function json(status, value, headers) {
+        send(
+          status,
+          JSON.stringify(value),
+          "application/json; charset=utf-8",
+          headers,
+        );
+      }
+      function error(status, title, message, headers) {
+        send(status, renderError({ title, message }), undefined, headers);
+      }
+      async function handle() {
+        const path = req.url;
+        if (path === "/check") {
+          if (req.method !== "POST")
+            return error(
+              405,
+              "Use the release form",
+              "A check starts only when you submit the form.",
+              { allow: "POST" },
+            );
+          if (
+            req.headers.origin &&
+            req.headers.origin !== `http://${req.headers.host}` &&
+            req.headers.origin !== `https://${req.headers.host}`
+          )
+            return error(
+              403,
+              "Request origin rejected",
+              "Open the release form on this site to start a check.",
+            );
+          if (req.headers["sec-fetch-site"] === "cross-site")
+            return error(
+              403,
+              "Request origin rejected",
+              "Open the release form on this site to start a check.",
+            );
+          if (
+            !/^application\/x-www-form-urlencoded(?:\s*;\s*charset=utf-8)?$/i.test(
+              req.headers["content-type"] || "",
+            )
+          )
+            return error(
+              415,
+              "Use the release form",
+              "Submit a package name and exact version with the release form.",
+            );
+          let body;
+          try {
+            body = await readBody(req, limits);
+          } catch (e) {
+            return error(
+              e.status || 400,
+              e.status === 413
+                ? "Request too large"
+                : "Request body unavailable",
+              "Submit only a package name and exact version.",
+            );
+          }
+          const params = new URLSearchParams(body);
+          const value = params.get("package") || "";
+          const name = params.size === 1 && parseNameAtExactVersion(value);
+          if (!name)
+            return send(
+              400,
+              renderHome({
+                value: value.slice(0, 300),
+                error:
+                  "Enter an npm package name and exact version. Tags, ranges and URLs are not accepted.",
+              }),
+            );
+          const result = queue.submit(name);
+          if (!result.job)
+            return send(
+              429,
+              renderHome({
+                value,
+                error:
+                  "The shared global limit is six new checks per ten minutes, with one active check and one waiting slot.",
+                retryAfter: result.retryAfter,
+              }),
+              undefined,
+              { "retry-after": String(result.retryAfter) },
+            );
+          return send(303, "", "text/plain; charset=utf-8", {
+            location: `/r/${result.job.id}`,
+            "x-unmask-reused": String(result.reused),
+          });
+        }
+        if (req.method !== "GET")
+          return error(
+            405,
+            "Method unavailable",
+            "Use the release form to start a check.",
+            { allow: "GET" },
+          );
+        if (path === "/") return send(200, renderHome());
+        if (path === "/healthz") return json(200, { status: "ok" });
+        if (path === "/version") return json(200, version);
+        const match = /^\/r\/([a-f0-9]{32})(\.json)?$/.exec(path);
+        if (!match)
+          return error(
+            404,
+            "Page unavailable",
+            "Return to the release form to check a package.",
+          );
+        const job = queue.get(match[1]);
+        if (!job)
+          return error(
+            404,
+            "Report unavailable",
+            "This report may have expired, been evicted or been lost in a restart. Submit the package again to request a new check.",
+          );
+        const pending = ["queued", "running"].includes(job.status);
+        if (match[2]) {
+          if (job.report)
+            return json(200, job.report, {
+              "content-disposition":
+                'attachment; filename="unmask-report.json"',
+            });
+          const { key, ...publicJob } = job;
+          return json(
+            pending ? 202 : 200,
+            publicJob,
+            pending ? { "retry-after": "3" } : {},
+          );
+        }
+        if (job.report)
+          return send(200, renderCompleted(renderReport(job.report), job));
+        return send(
+          200,
+          renderStatus(job),
+          undefined,
+          pending ? { "retry-after": "3" } : {},
+        );
+      }
+      handle().catch(() =>
+        error(
+          500,
+          "Request could not finish",
+          "Return to the release form to try again.",
+        ),
+      );
+    },
+  );
+  server.on("close", () => queue.close());
+  return server;
+}

package/dist/server/main.mjs

@@ -0,0 +1,146 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import os from "node:os";
+import { pathToFileURL } from "node:url";
+import { createApp } from "./app.mjs";
+import { createRunner } from "./runner.mjs";
+import { modelClientFromEnv } from "../model/config.mjs";
+export { modelClientFromEnv };
+
+export async function loadVersionInfo({
+  metadataURL = new URL("../version.json", import.meta.url),
+  tarballSha256 = process.env.UNMASK_TARBALL_SHA256,
+} = {}) {
+  const metadata = JSON.parse(
+    await readFile(new URL("../../package.json", import.meta.url), "utf8"),
+  );
+  let commit = "unknown";
+  try {
+    const build = JSON.parse(await readFile(metadataURL, "utf8"));
+    if (
+      build.version !== metadata.version ||
+      typeof build.dirty !== "boolean" ||
+      !/^(?:[a-f0-9]{40}|unknown)$/.test(build.commit) ||
+      !/^[a-f0-9]{64}$/.test(build.sourceSha256)
+    )
+      throw new TypeError("Invalid build version metadata");
+    if (!build.dirty) commit = build.commit;
+  } catch (error) {
+    if (error.code !== "ENOENT") throw error;
+  }
+  if (tarballSha256 !== undefined && !/^[a-f0-9]{64}$/.test(tarballSha256))
+    throw new TypeError("Invalid tarball digest");
+  return {
+    version: metadata.version,
+    commit,
+    tarballSha256: tarballSha256 || "unknown",
+  };
+}
+
+export async function startServer({
+  host = process.env.HOST || "127.0.0.1",
+  port = Number(process.env.PORT || 3000),
+  tempRoot = path.join(
+    os.tmpdir(),
+    `unmask-server-${process.getuid?.() ?? "local"}`,
+  ),
+  runnerOptions = {},
+  versionInfo,
+  modelClient,
+} = {}) {
+  if (
+    typeof host !== "string" ||
+    !host ||
+    !Number.isSafeInteger(port) ||
+    port < 0 ||
+    port > 65535
+  )
+    throw new TypeError("Invalid server address");
+  const client =
+    modelClient === null
+      ? undefined
+      : modelClient === undefined
+        ? await modelClientFromEnv()
+        : modelClient;
+  const run = await createRunner({
+    ...runnerOptions,
+    tempRoot,
+    modelClient: client,
+  });
+  const shutdown = new AbortController();
+  const pending = new Set();
+  const runAnalysis = (spec, { signal } = {}) => {
+    const operation = run(spec, {
+      signal: signal
+        ? AbortSignal.any([signal, shutdown.signal])
+        : shutdown.signal,
+    });
+    pending.add(operation);
+    operation.finally(() => pending.delete(operation)).catch(() => {});
+    return operation;
+  };
+  const server = createApp({
+    runAnalysis,
+    versionInfo: {
+      ...(versionInfo || (await loadVersionInfo())),
+      modelConfigDigest:
+        client?.configHash || versionInfo?.modelConfigDigest || "disabled",
+    },
+  });
+  let closing;
+  function close() {
+    closing ||= (async () => {
+      shutdown.abort();
+      const stopped = new Promise((resolve, reject) => {
+        server.close((error) =>
+          error && error.code !== "ERR_SERVER_NOT_RUNNING"
+            ? reject(error)
+            : resolve(),
+        );
+      });
+      server.closeIdleConnections();
+      await Promise.all([
+        stopped,
+        ...Array.from(pending, (operation) => operation.catch(() => {})),
+      ]);
+    })();
+    return closing;
+  }
+  try {
+    await new Promise((resolve, reject) => {
+      server.once("error", reject);
+      server.listen(port, host, () => {
+        server.removeListener("error", reject);
+        resolve();
+      });
+    });
+  } catch (error) {
+    await close();
+    throw error;
+  }
+  return { server, close };
+}
+
+async function main() {
+  const runtime = await startServer();
+  const stop = async () => {
+    try {
+      await runtime.close();
+    } catch {
+      process.exitCode = 1;
+    }
+  };
+  process.once("SIGINT", stop);
+  process.once("SIGTERM", stop);
+  const address = runtime.server.address();
+  console.log(`UNMASK listening on ${address.address}:${address.port}`);
+}
+
+if (
+  process.argv[1] &&
+  pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url
+)
+  main().catch(() => {
+    console.error("UNMASK server could not start");
+    process.exitCode = 1;
+  });

package/dist/server/queue.mjs

@@ -0,0 +1,177 @@
+import { randomBytes } from "node:crypto";
+
+export const DEFAULT_LIMITS = Object.freeze({
+  maxBodyBytes: 4096,
+  bodyTimeoutMs: 10000,
+  queueTtlMs: 300000,
+  executionMs: 240000,
+  maxNewChecks: 6,
+  admissionWindowMs: 600000,
+  maxReportBytes: 64 * 1024 * 1024,
+  maxReports: 200,
+  reportTtlMs: 1800000,
+});
+export const systemClock = { now: Date.now, setTimeout, clearTimeout };
+
+export function resolveLimits(overrides = {}) {
+  for (const [key, value] of Object.entries(overrides)) {
+    if (
+      !(key in DEFAULT_LIMITS) ||
+      !Number.isSafeInteger(value) ||
+      value < 1 ||
+      value > DEFAULT_LIMITS[key]
+    )
+      throw new TypeError(`Invalid limit: ${key}`);
+  }
+  return { ...DEFAULT_LIMITS, ...overrides };
+}
+
+export function createQueue({
+  runAnalysis,
+  store,
+  limits,
+  clock = systemClock,
+  keyPrefix = "",
+}) {
+  let active,
+    waiting,
+    closed = false;
+  let admissions = [];
+  function persist(job) {
+    if (!store.put(job)) {
+      const { report, ...small } = job;
+      store.put({ ...small, status: "incomplete", reason: "report-too-large" });
+    }
+  }
+  function expireWaiting(job) {
+    if (waiting !== job) return;
+    waiting = undefined;
+    job.status = "incomplete";
+    job.reason = "queue-expired";
+    job.finishedAt = clock.now();
+    clock.clearTimeout(job.queueTimer);
+    delete job.queueTimer;
+    persist(job);
+  }
+  function begin(job) {
+    active = job;
+    waiting = undefined;
+    if (job.queueTimer !== undefined) clock.clearTimeout(job.queueTimer);
+    delete job.queueTimer;
+    job.status = "running";
+    job.startedAt = clock.now();
+    const controller = new AbortController();
+    const timer = clock.setTimeout(() => {
+      job.status = "incomplete";
+      job.reason = "execution-timeout";
+      job.finishedAt = clock.now();
+      controller.abort();
+      persist(job);
+    }, limits.executionMs);
+    timer?.unref?.();
+    job.cancel = () => {
+      clock.clearTimeout(timer);
+      controller.abort();
+    };
+    Promise.resolve()
+      .then(() =>
+        runAnalysis(job.nameAtExactVersion, { signal: controller.signal }),
+      )
+      .then((report) => {
+        if (job.status !== "running" || closed) return;
+        if (
+          !report ||
+          typeof report !== "object" ||
+          Array.isArray(report) ||
+          ![0, 2, 3].includes(report.policy?.exitCode)
+        )
+          throw new TypeError("Invalid report");
+        const status = report.policy.exitCode === 2 ? "incomplete" : "done";
+        const finishedAt = clock.now();
+        persist({ ...publicJob(job), status, report, finishedAt });
+        job.status = status;
+        job.finishedAt = finishedAt;
+      })
+      .catch(() => {
+        if (job.status !== "running" || closed) return;
+        job.status = "failed";
+        job.reason = "analysis-failed";
+        job.finishedAt = clock.now();
+        delete job.report;
+        persist(publicJob(job));
+      })
+      .finally(() => {
+        clock.clearTimeout(timer);
+        active = undefined;
+        if (waiting && !closed) {
+          if (clock.now() - waiting.createdAt >= limits.queueTtlMs)
+            expireWaiting(waiting);
+          else begin(waiting);
+        }
+      });
+  }
+  function publicJob(job) {
+    const { cancel, queueTimer, report, ...metadata } = job;
+    return report ? { ...metadata, report } : metadata;
+  }
+  return {
+    submit(nameAtExactVersion) {
+      const key = `${keyPrefix}\n${nameAtExactVersion}`;
+      const existing =
+        [active, waiting].find(
+          (job) =>
+            job?.key === key && ["queued", "running"].includes(job.status),
+        ) || store.findKey(key);
+      if (existing) return { job: publicJob(existing), reused: true };
+      admissions = admissions.filter(
+        (time) => clock.now() - time < limits.admissionWindowMs,
+      );
+      if (
+        closed ||
+        (active && waiting) ||
+        admissions.length >= limits.maxNewChecks
+      )
+        return {
+          retryAfter: Math.max(
+            1,
+            Math.ceil(
+              (admissions.length >= limits.maxNewChecks
+                ? admissions[0] + limits.admissionWindowMs - clock.now()
+                : limits.executionMs) / 1000,
+            ),
+          ),
+        };
+      const job = {
+        id: randomBytes(16).toString("hex"),
+        key,
+        nameAtExactVersion,
+        status: "queued",
+        createdAt: clock.now(),
+      };
+      admissions.push(clock.now());
+      if (!active) begin(job);
+      else {
+        waiting = job;
+        job.queueTimer = clock.setTimeout(
+          () => expireWaiting(job),
+          limits.queueTtlMs,
+        );
+      }
+      return { job: publicJob(job), reused: false };
+    },
+    get(id) {
+      const job = [active, waiting].find((item) => item?.id === id);
+      return job && ["queued", "running"].includes(job.status)
+        ? publicJob(job)
+        : store.get(id);
+    },
+    close() {
+      closed = true;
+      if (waiting) {
+        clock.clearTimeout(waiting.queueTimer);
+        waiting = undefined;
+      }
+      active?.cancel();
+    },
+  };
+}