@sudobility/sider_lib 0.0.2

package/src/recipe.ts

@@ -0,0 +1,143 @@
+// Recipe compiler: ToolSpec + args → CompiledRequest (secrets still placeholders).
+//
+// Secrets are compiled to an internal SENTINEL (a Private-Use-Area delimited
+// token that cannot occur in real data), NOT to their values. The sentinel
+// survives through the egress gates and is only replaced with a real value at
+// detokenization (see execute.ts), after all three gates pass.
+
+import type {
+  CompiledRequest,
+  InvocationRecipe,
+  ParamBinding,
+  ToolSpec,
+} from "@sudobility/sider_types";
+
+const S = ""; // sentinel delimiter (PUA)
+
+export function secretSentinel(slotId: string): string {
+  return `${S}${slotId}${S}`;
+}
+
+export class RecipeCompileError extends Error {}
+
+export interface CompileContext {
+  /** Origin the recipe runs against (path templates are relative to it). */
+  siteOrigin: string;
+  /** User/LLM-supplied tool arguments. */
+  args?: Record<string, unknown>;
+  /** Prior tool results this run, keyed by ComposeStep.ref (for priorOutput bindings). */
+  priorOutputs?: Record<string, unknown>;
+}
+
+export function compileRecipe(tool: ToolSpec, ctx: CompileContext): CompiledRequest {
+  const recipe = tool.recipe;
+  const referenced = new Set<string>();
+
+  const resolveKey = (key: string): string => {
+    const b = recipe.bindings[key];
+    if (!b) throw new RecipeCompileError(`Unbound placeholder {${key}} in tool "${tool.name}"`);
+    return resolveBinding(key, b, ctx, referenced);
+  };
+
+  const path = fillTemplate(recipe.urlTemplate, resolveKey);
+  const url = joinOrigin(ctx.siteOrigin, path);
+
+  const headers: Record<string, string> = {};
+  for (const h of recipe.headers) headers[h.name] = fillTemplate(h.valueTemplate, resolveKey);
+
+  const body =
+    recipe.bodyTemplate !== undefined ? resolveDeep(recipe.bodyTemplate, resolveKey) : undefined;
+
+  return {
+    method: recipe.method,
+    url,
+    headers,
+    body,
+    credentials: "include",
+    referencedSlotIds: [...referenced],
+    safetyClass: tool.safetyClass,
+  };
+}
+
+function resolveBinding(
+  key: string,
+  b: ParamBinding,
+  ctx: CompileContext,
+  referenced: Set<string>,
+): string {
+  switch (b.kind) {
+    case "literal":
+      return b.value;
+    case "arg": {
+      const v = ctx.args?.[b.argName];
+      if (v === undefined) throw new RecipeCompileError(`Missing arg "${b.argName}" for {${key}}`);
+      return String(v);
+    }
+    case "secret":
+      referenced.add(b.slotId);
+      return secretSentinel(b.slotId);
+    case "priorOutput": {
+      const v = getByPath(ctx.priorOutputs?.[b.stepRef], b.jsonPath);
+      if (v === undefined) {
+        throw new RecipeCompileError(`priorOutput ${b.stepRef}.${b.jsonPath} unresolved for {${key}}`);
+      }
+      return String(v);
+    }
+  }
+}
+
+function fillTemplate(tmpl: string, resolve: (key: string) => string): string {
+  return tmpl.replace(/\{([^{}]+)\}/g, (_m, key: string) => resolve(key));
+}
+
+function resolveDeep(node: unknown, resolve: (key: string) => string): unknown {
+  if (typeof node === "string") return fillTemplate(node, resolve);
+  if (Array.isArray(node)) return node.map((n) => resolveDeep(n, resolve));
+  if (node && typeof node === "object") {
+    const out: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(node)) out[k] = resolveDeep(v, resolve);
+    return out;
+  }
+  return node;
+}
+
+function joinOrigin(origin: string, path: string): string {
+  if (/^https?:\/\//i.test(path)) return path; // absolute (still same-origin-gated)
+  return `${origin.replace(/\/$/, "")}/${path.replace(/^\//, "")}`;
+}
+
+/** Read a dotted/bracketed path from an object: "data.sections[0].id". */
+export function getByPath(obj: unknown, path: string): unknown {
+  const parts = path
+    .replace(/\[(\d+)\]/g, ".$1")
+    .split(".")
+    .filter(Boolean);
+  let cur: unknown = obj;
+  for (const p of parts) {
+    if (cur == null || typeof cur !== "object") return undefined;
+    cur = (cur as Record<string, unknown>)[p];
+  }
+  return cur;
+}
+
+// --- Recipe canonicalization for corroboration hashing --------------------
+
+/** Stable stringify (sorted keys) so two users' equivalent recipes hash equal. */
+export function canonicalizeRecipe(recipe: InvocationRecipe): string {
+  return stableStringify(recipe);
+}
+
+export function hashRecipe(recipe: InvocationRecipe): string {
+  const s = canonicalizeRecipe(recipe);
+  let h = 0;
+  for (let i = 0; i < s.length; i++) h = (Math.imul(31, h) + s.charCodeAt(i)) | 0;
+  return (h >>> 0).toString(16);
+}
+
+function stableStringify(v: unknown): string {
+  if (v === null || typeof v !== "object") return JSON.stringify(v) ?? "null";
+  if (Array.isArray(v)) return `[${v.map(stableStringify).join(",")}]`;
+  const obj = v as Record<string, unknown>;
+  const keys = Object.keys(obj).sort();
+  return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(obj[k])}`).join(",")}}`;
+}

package/src/templating.ts

@@ -0,0 +1,69 @@
+// Deterministic clustering & path templating. Runs BEFORE any AI: groups raw
+// observations into endpoint templates so the brain infers semantics once per
+// endpoint, not once per request. (This is the logic distill.ts inlined.)
+
+import type { Observation } from "@sudobility/sider_types";
+
+const UUID_RE =
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const LONGHEX_RE = /^[0-9a-f]{16,}$/i;
+
+/** Replace id-like path segments with `{id}`: /sections/120/seats → /sections/{id}/seats */
+export function templatePath(rawUrl: string): string {
+  let pathname = rawUrl;
+  try {
+    pathname = new URL(rawUrl, "http://sider.local").pathname;
+  } catch {
+    /* keep raw */
+  }
+  return pathname
+    .split("/")
+    .map((seg) => {
+      if (seg === "") return seg;
+      if (/^\d+$/.test(seg)) return "{id}";
+      if (UUID_RE.test(seg)) return "{id}";
+      if (LONGHEX_RE.test(seg)) return "{id}";
+      return seg;
+    })
+    .join("/");
+}
+
+/**
+ * GraphQL keying: one URL serves many operations, so cluster on operation name,
+ * not path. Returns undefined for non-GraphQL requests.
+ */
+export function graphqlOperationOf(url: string, body: unknown): string | undefined {
+  if (body && typeof body === "object") {
+    const b = body as Record<string, unknown>;
+    if (typeof b["operationName"] === "string" && b["operationName"]) {
+      return b["operationName"];
+    }
+    if (typeof b["query"] === "string") {
+      const m = /\b(query|mutation|subscription)\s+([A-Za-z0-9_]+)/.exec(b["query"]);
+      if (m && m[2]) return m[2];
+    }
+  }
+  if (/\/graphql\b/i.test(url)) return "anonymous";
+  return undefined;
+}
+
+type Clusterable = Pick<Observation, "method" | "url" | "requestBody">;
+
+/** Stable key identifying the endpoint template an observation belongs to. */
+export function endpointKey(o: Clusterable): string {
+  const op = graphqlOperationOf(o.url, o.requestBody);
+  return `${o.method} ${templatePath(o.url)}${op ? ` #${op}` : ""}`;
+}
+
+export function clusterObservations<T extends Clusterable>(
+  observations: T[],
+): Map<string, T[]> {
+  const groups = new Map<string, T[]>();
+  for (const o of observations) {
+    const key = endpointKey(o);
+    const g = groups.get(key);
+    if (g) g.push(o);
+    else groups.set(key, [o]);
+  }
+  return groups;
+}

package/src/tokenize.test.ts

@@ -0,0 +1,54 @@
+import { test, expect } from "bun:test";
+import { tokenizeObservation } from "./tokenize";
+import type { RawObservation, KnownSecret } from "./tokenize";
+
+function raw(overrides: Partial<RawObservation> = {}): RawObservation {
+  return {
+    method: "GET",
+    url: "https://resale.fifa.com/api/seats",
+    requestHeaders: {},
+    requestBody: undefined,
+    status: 200,
+    responseBody: undefined,
+    timingMs: 10,
+    context: { route: "/seats" },
+    ...overrides,
+  };
+}
+
+test("masks a credential-named request header", () => {
+  const { observation, slots } = tokenizeObservation(
+    raw({ requestHeaders: { authorization: "Bearer abc.def.ghi" } }),
+    [],
+  );
+  expect(JSON.stringify(observation.requestHeaders)).not.toContain("abc.def.ghi");
+  expect(slots.length).toBeGreaterThan(0);
+});
+
+test("masks a known cookie/storage value wherever it appears in a body", () => {
+  const known: KnownSecret[] = [
+    {
+      value: "SUPERSECRETVALUE123",
+      slotId: "slot:resale.fifa.com:session",
+      kind: "session",
+      role: "active_inject",
+      resolverHint: { from: "cookie", name: "sid" },
+    },
+  ];
+  const { observation } = tokenizeObservation(
+    raw({ responseBody: { echoed: "SUPERSECRETVALUE123", ok: true } }),
+    known,
+  );
+  expect(JSON.stringify(observation.responseBody)).not.toContain("SUPERSECRETVALUE123");
+});
+
+test("masks a JWT-shaped string in a body", () => {
+  const jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.abc123signature";
+  const { observation } = tokenizeObservation(raw({ responseBody: { token: jwt } }), []);
+  expect(JSON.stringify(observation.responseBody)).not.toContain(jwt);
+});
+
+test("leaves non-secret data untouched", () => {
+  const { observation } = tokenizeObservation(raw({ responseBody: { price: 400, section: "A1" } }), []);
+  expect(observation.responseBody).toEqual({ price: 400, section: "A1" });
+});

package/src/tokenize.ts

@@ -0,0 +1,242 @@
+// Tokenizer (browser-side, but PURE here — the caller supplies the secret
+// values it discovered in the user's own cookie jar / storage). Replaces secrets
+// in a raw observation with a readable placeholder "{{secret:<slotId>}}" and
+// emits SecretSlot proposals — so nothing with a real secret leaves the browser.
+//
+// Coverage (defense in depth, because storage cross-reference alone misses
+// secrets minted this session or carried in bodies / non-standard headers):
+//   1. Known values gathered from the user's own cookies/storage (exact match).
+//   2. Any request header whose NAME looks credential-bearing.
+//   3. Body fields whose KEY looks credential-bearing (e.g. access_token).
+//   4. JWT-shaped strings anywhere in a body.
+// Bias is conservative-for-privacy: over-mask rather than leak.
+
+import type {
+  HttpMethod,
+  InjectionLocation,
+  Observation,
+  RequestContext,
+  ResolverHint,
+  SecretKind,
+  SecretRole,
+  SecretSlot,
+} from "@sudobility/sider_types";
+
+export interface RawObservation {
+  method: HttpMethod;
+  url: string;
+  requestHeaders: Record<string, string>;
+  requestBody?: unknown;
+  status: number;
+  responseBody?: unknown;
+  timingMs: number;
+  context: RequestContext;
+}
+
+/** A secret value the browser found in THIS user's session (value stays local). */
+export interface KnownSecret {
+  value: string;
+  slotId: string; // stable placeholder, e.g. "slot:example.com:auth_bearer"
+  kind: SecretKind;
+  role: SecretRole;
+  resolverHint: ResolverHint;
+}
+
+export type SecretSlotProposal = Omit<SecretSlot, "siteId" | "createdAt" | "updatedAt">;
+export type TokenizedObservation = Omit<Observation, "id" | "batchId" | "createdAt">;
+
+export interface TokenizeResult {
+  observation: TokenizedObservation;
+  slots: SecretSlotProposal[];
+}
+
+export function samplePlaceholder(slotId: string): string {
+  return `{{secret:${slotId}}}`;
+}
+
+// Header NAME looks credential-bearing (conservative-for-privacy: any "key",
+// "token", "auth", "secret", "session", etc. in the name).
+const SECRET_HEADER_RE = /authorization|cookie|token|key|secret|session|auth|xsrf|csrf|bearer/i;
+// Body property KEY looks credential-bearing.
+const SECRET_KEY_RE =
+  /(^|[_-])(access_token|refresh_token|id_token|token|api[-_]?key|apikey|client_secret|secret|password|passwd|session_?(id|token)|jwt|bearer|credential|auth_?token)($|[_-])/i;
+// JWT-shaped string.
+const JWT_RE = /eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{2,}/g;
+const MIN_MASK_LEN = 6;
+
+export function tokenizeObservation(raw: RawObservation, known: KnownSecret[]): TokenizeResult {
+  const origin = originOf(raw.url);
+  const host = hostOf(origin);
+  const slots = new Map<string, SecretSlotProposal>();
+
+  let url = raw.url;
+  const headers: Record<string, string> = { ...raw.requestHeaders };
+  let requestBody = raw.requestBody;
+  let responseBody = raw.responseBody;
+
+  const ensure = (slotId: string, kind: SecretKind, role: SecretRole, loc: InjectionLocation, hint?: ResolverHint) => {
+    if (slots.has(slotId)) return;
+    slots.set(slotId, {
+      id: slotId,
+      kind,
+      role,
+      injectionLocation: loc,
+      allowedDestination: origin,
+      resolverHints: hint ? [hint] : [],
+      confidence: 0.6,
+    });
+  };
+
+  // 1. Known values (exact substring match everywhere).
+  for (const s of known) {
+    if (!s.value) continue;
+    const loc = locate(raw, s.value) ?? { at: "header" as const, name: "authorization" };
+    const ph = samplePlaceholder(s.slotId);
+    url = url.split(s.value).join(ph);
+    for (const k of Object.keys(headers)) headers[k] = headers[k]!.split(s.value).join(ph);
+    requestBody = replaceInJson(requestBody, s.value, ph);
+    responseBody = replaceInJson(responseBody, s.value, ph);
+    ensure(s.slotId, s.kind, s.role, loc, s.resolverHint);
+  }
+
+  // 2. Credential-bearing headers (by name), even when the value wasn't in storage.
+  for (const name of Object.keys(headers)) {
+    const val = headers[name];
+    if (typeof val !== "string" || !val || val.startsWith("{{secret:")) continue;
+    if (!SECRET_HEADER_RE.test(name)) continue;
+    const lname = name.toLowerCase();
+    const slotId = `slot:${host}:${normalize(name)}`;
+    headers[name] = samplePlaceholder(slotId);
+    ensure(
+      slotId,
+      guessKind(name),
+      lname === "cookie" ? "auto_cookie" : "active_inject",
+      lname === "cookie" ? { at: "cookie", name } : { at: "header", name },
+    );
+  }
+
+  // 3 + 4. Body fields by key + JWTs anywhere.
+  requestBody = maskBody(requestBody, host, origin, slots, ensure, "");
+  responseBody = maskBody(responseBody, host, origin, slots, ensure, "");
+
+  return {
+    observation: {
+      method: raw.method,
+      url,
+      requestHeaders: headers,
+      requestBody,
+      status: raw.status,
+      responseBody,
+      timingMs: raw.timingMs,
+      context: raw.context,
+    },
+    slots: [...slots.values()],
+  };
+}
+
+type Ensure = (
+  slotId: string,
+  kind: SecretKind,
+  role: SecretRole,
+  loc: InjectionLocation,
+  hint?: ResolverHint,
+) => void;
+
+function maskBody(node: unknown, host: string, origin: string, slots: Map<string, SecretSlotProposal>, ensure: Ensure, path: string, key?: string): unknown {
+  if (node === undefined || node === null) return node;
+  if (typeof node === "string") {
+    if (key && node.length >= MIN_MASK_LEN && SECRET_KEY_RE.test(key)) {
+      const slotId = `slot:${host}:${normalize(key)}`;
+      ensure(slotId, guessKind(key), "active_inject", { at: "body", jsonPath: path });
+      return samplePlaceholder(slotId);
+    }
+    if (JWT_RE.test(node)) {
+      JWT_RE.lastIndex = 0;
+      const slotId = `slot:${host}:jwt`;
+      ensure(slotId, "bearer", "active_inject", { at: "body", jsonPath: path });
+      return node.replace(JWT_RE, samplePlaceholder(slotId));
+    }
+    return node;
+  }
+  if (Array.isArray(node)) return node.map((n, i) => maskBody(n, host, origin, slots, ensure, `${path}[${i}]`, key));
+  if (typeof node === "object") {
+    const out: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(node)) out[k] = maskBody(v, host, origin, slots, ensure, path ? `${path}.${k}` : k, k);
+    return out;
+  }
+  return node;
+}
+
+// --- helpers ---------------------------------------------------------------
+
+function guessKind(name: string): SecretKind {
+  const n = name.toLowerCase();
+  if (n.includes("csrf") || n.includes("xsrf")) return "csrf";
+  if (n.includes("cookie") || n.includes("session")) return "session";
+  if (n.includes("auth") || n.includes("bearer") || n.includes("token") || n.includes("jwt")) return "bearer";
+  if (n.includes("api") && n.includes("key")) return "api_key";
+  return "unknown";
+}
+
+function locate(raw: RawObservation, value: string): InjectionLocation | null {
+  for (const [name, v] of Object.entries(raw.requestHeaders)) {
+    if (typeof v === "string" && v.includes(value)) return { at: "header", name };
+  }
+  try {
+    const u = new URL(raw.url, "http://sider.local");
+    for (const [param, v] of u.searchParams) if (v.includes(value)) return { at: "query", param };
+  } catch {
+    /* ignore */
+  }
+  const bodyPath = findInJson(raw.requestBody, value);
+  if (bodyPath) return { at: "body", jsonPath: bodyPath };
+  return null;
+}
+
+function findInJson(node: unknown, value: string, prefix = ""): string | null {
+  if (typeof node === "string") return node.includes(value) ? prefix.replace(/^\./, "") : null;
+  if (Array.isArray(node)) {
+    for (let i = 0; i < node.length; i++) {
+      const p = findInJson(node[i], value, `${prefix}[${i}]`);
+      if (p) return p;
+    }
+    return null;
+  }
+  if (node && typeof node === "object") {
+    for (const [k, v] of Object.entries(node)) {
+      const p = findInJson(v, value, `${prefix}.${k}`);
+      if (p) return p;
+    }
+  }
+  return null;
+}
+
+function replaceInJson(node: unknown, value: string, ph: string): unknown {
+  if (node === undefined) return node;
+  if (typeof node === "string") return node.split(value).join(ph);
+  if (Array.isArray(node)) return node.map((n) => replaceInJson(n, value, ph));
+  if (node && typeof node === "object") {
+    const out: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(node)) out[k] = replaceInJson(v, value, ph);
+    return out;
+  }
+  return node;
+}
+
+function originOf(url: string): string {
+  try {
+    return new URL(url).origin;
+  } catch {
+    return url;
+  }
+}
+function hostOf(origin: string): string {
+  try {
+    return new URL(origin).host;
+  } catch {
+    return origin;
+  }
+}
+function normalize(s: string): string {
+  return s.toLowerCase().replace(/[^a-z0-9]+/g, "_");
+}