fullstackgtm 0.25.2 → 0.26.0

package/src/integrity.ts

@@ -0,0 +1,146 @@
+import { createHmac, randomBytes } from "node:crypto";
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { credentialsDir, ensureSecureHomeDir, writeSecureFile } from "./credentials.ts";
+import type { PatchOperation } from "./types.ts";
+
+/**
+ * Approval integrity.
+ *
+ * The plan store records WHICH operation ids a human approved, but the apply
+ * path re-reads the operation BODIES fresh from the (user-editable) plan file.
+ * Nothing bound the approval to the content: an approved op's afterValue or
+ * objectId could be changed on disk between `plans approve` and `apply` — by a
+ * compromised dependency, a co-tenant, or a plan file synced/edited on another
+ * machine — and the changed value would be written under the prior approval.
+ *
+ * Fix: at approval time, HMAC-sign each approved operation's security-relevant
+ * content (including the approved value override) with a per-install secret key
+ * stored 0600 alongside the credentials. At apply time, recompute and verify.
+ * Any post-approval edit to the operations or the approved overrides changes the
+ * signature; a tamper must now also forge an HMAC it cannot compute without the
+ * key. The key never leaves the machine, so a plan approved here and applied
+ * elsewhere fails closed ("re-approve on this machine") rather than open.
+ *
+ * This raises the bar from "trust the plan JSON" to "trust the plan JSON only
+ * insofar as it still matches what was signed with the local key." It is not a
+ * defense against an attacker who already holds the signing key (same-dir, same
+ * permissions as the credential store) — that is the documented boundary.
+ */
+
+const SIGNING_KEY_FILE = ".plan-signing-key";
+
+function signingKeyPath(): string {
+  return join(credentialsDir(), SIGNING_KEY_FILE);
+}
+
+/** Read the signing key, or null if it has not been created yet. */
+export function loadSigningKey(): Buffer | null {
+  const path = signingKeyPath();
+  if (!existsSync(path)) return null;
+  try {
+    return Buffer.from(readFileSync(path, "utf8").trim(), "hex");
+  } catch {
+    return null;
+  }
+}
+
+/** Read the signing key, creating a fresh 32-byte one (0600) on first use. */
+export function loadOrCreateSigningKey(): Buffer {
+  const existing = loadSigningKey();
+  if (existing && existing.length >= 32) return existing;
+  ensureSecureHomeDir();
+  const key = randomBytes(32);
+  writeSecureFile(signingKeyPath(), `${key.toString("hex")}\n`);
+  return key;
+}
+
+/**
+ * Canonical, stable string of the operation content an approval binds to. Only
+ * the fields that determine WHAT gets written: changing any of them must
+ * invalidate the approval. `override` is the approved value override for this op
+ * (the value actually written when set), so tampering with stored overrides is
+ * caught too.
+ */
+function canonicalApprovalContent(operation: PatchOperation, override: unknown): string {
+  return JSON.stringify([
+    operation.id,
+    operation.operation,
+    operation.objectType,
+    operation.objectId,
+    operation.field ?? null,
+    operation.beforeValue ?? null,
+    operation.afterValue ?? null,
+    operation.groupId ?? null,
+    // Safety-relevant fields too: editing a precondition could relax a drift
+    // guard, and forging forceArchiveDuplicate could suppress the archive-of-
+    // duplicate refusal — the signed approval must pin apply BEHAVIOR, not just
+    // the written value. `reason` is human-reviewed AND written verbatim into
+    // create_task bodies (afterValue ?? reason fallback in the connectors), so a
+    // create_task with a null afterValue would otherwise let a disk edit to
+    // reason write unapproved text under a still-valid digest.
+    operation.preconditions ?? null,
+    operation.forceArchiveDuplicate ?? false,
+    operation.reason ?? null,
+    override === undefined ? null : ["__override__", override],
+  ]);
+}
+
+/** HMAC-SHA256 signature of one operation's approved content. */
+export function signApproval(operation: PatchOperation, override: unknown, key: Buffer): string {
+  return createHmac("sha256", key).update(canonicalApprovalContent(operation, override)).digest("hex");
+}
+
+/**
+ * Compute the approval signature map for a set of approved operation ids,
+ * resolving each op from the plan and its (approved) value override.
+ */
+export function computeApprovalDigests(
+  operations: PatchOperation[],
+  approvedOperationIds: string[],
+  valueOverrides: Record<string, unknown>,
+  key: Buffer,
+): Record<string, string> {
+  const byId = new Map(operations.map((operation) => [operation.id, operation]));
+  const digests: Record<string, string> = {};
+  for (const id of approvedOperationIds) {
+    const operation = byId.get(id);
+    if (!operation) continue;
+    digests[id] = signApproval(operation, valueOverrides[id], key);
+  }
+  return digests;
+}
+
+export type ApprovalVerification =
+  | { ok: true }
+  | { ok: false; reason: "no_key"; tampered: string[] }
+  | { ok: false; reason: "mismatch"; tampered: string[] };
+
+/**
+ * Verify that every approved operation still matches what was signed. Returns
+ * ok:true when there are no stored digests (a pre-integrity plan — nothing to
+ * verify), when all match, or fails with the list of operation ids whose
+ * content changed since approval.
+ */
+export function verifyApprovalDigests(
+  operations: PatchOperation[],
+  approvedOperationIds: string[],
+  valueOverrides: Record<string, unknown>,
+  storedDigests: Record<string, string> | undefined,
+): ApprovalVerification {
+  if (!storedDigests || Object.keys(storedDigests).length === 0) return { ok: true };
+  const key = loadSigningKey();
+  if (!key) return { ok: false, reason: "no_key", tampered: approvedOperationIds };
+  const byId = new Map(operations.map((operation) => [operation.id, operation]));
+  const tampered: string[] = [];
+  for (const id of approvedOperationIds) {
+    const operation = byId.get(id);
+    const expected = storedDigests[id];
+    if (!operation || !expected) {
+      tampered.push(id);
+      continue;
+    }
+    if (signApproval(operation, valueOverrides[id], key) !== expected) tampered.push(id);
+  }
+  return tampered.length === 0 ? { ok: true } : { ok: false, reason: "mismatch", tampered };
+}

package/src/marketTaxonomy.ts

@@ -0,0 +1,288 @@
+import {
+  DEFAULT_MODELS,
+  forcedToolCall,
+  type LlmCallOptions,
+} from "./llm.ts";
+import {
+  captureMarket,
+  type FetchPage,
+  loadCaptureTexts,
+  type MarketClaim,
+  type MarketConfig,
+  type MarketVendor,
+} from "./market.ts";
+
+/**
+ * Cold-start taxonomy bootstrap. `market init` writes a stub for a human
+ * analyst to fill in; the self-serve hosted map has no analyst in the loop, so
+ * this proposes the claim taxonomy automatically from the seed vendors' own
+ * pages.
+ *
+ * Posture matches the rest of the market layer: the LLM is a *proposal* layer
+ * grounded in captured evidence (it only sees text we actually fetched), and
+ * everything downstream — capture, classify with verbatim-span verification,
+ * front states, the report — stays deterministic over the stored observations.
+ * The taxonomy it emits is a normal `market.config.json` a human can still edit.
+ */
+
+export type SeedVendor = {
+  url: string;
+  /** Display name; derived from the host when omitted. */
+  name?: string;
+  /** Marks the user's own company as the anchor vendor. */
+  anchor?: boolean;
+};
+
+export type SuggestTaxonomyOptions = {
+  category: string;
+  vendors: SeedVendor[];
+  llm: LlmCallOptions;
+  /** Upper bound on proposed claims, to keep classification bounded. */
+  maxClaims?: number;
+  /** Per-vendor captured-text budget fed to the proposer (chars). */
+  perVendorChars?: number;
+  /** Test injectables. */
+  fetchPage?: FetchPage;
+  capturesDir?: string;
+  now?: () => Date;
+};
+
+export type SuggestTaxonomyResult = {
+  config: MarketConfig;
+  /** Vendors whose homepage capture was empty/failed (excluded from grounding). */
+  unreadableVendorIds: string[];
+  model: string;
+};
+
+const DEFAULT_MAX_CLAIMS = 16;
+const DEFAULT_PER_VENDOR_CHARS = 6_000;
+
+/** Stable, human-readable id from a string (claim capability or host). */
+function slugify(value: string, maxWords = 6): string {
+  const slug = value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .split("-")
+    .filter(Boolean)
+    .slice(0, maxWords)
+    .join("-");
+  return slug || "item";
+}
+
+/** Second-level domain as a vendor id seed: https://www.stripe.com/ -> stripe. */
+function vendorIdFromUrl(url: string): string {
+  let host: string;
+  try {
+    host = new URL(url).hostname;
+  } catch {
+    return slugify(url);
+  }
+  const labels = host.replace(/^www\./, "").split(".");
+  const sld = labels.length >= 2 ? labels[labels.length - 2] : labels[0];
+  return slugify(sld || host);
+}
+
+/** Disambiguate repeated ids by suffixing -2, -3, … */
+function uniqueId(base: string, taken: Set<string>): string {
+  if (!taken.has(base)) {
+    taken.add(base);
+    return base;
+  }
+  for (let n = 2; ; n += 1) {
+    const candidate = `${base}-${n}`;
+    if (!taken.has(candidate)) {
+      taken.add(candidate);
+      return candidate;
+    }
+  }
+}
+
+function provisionalVendors(seeds: SeedVendor[]): MarketVendor[] {
+  const taken = new Set<string>();
+  return seeds.map((seed) => {
+    const id = uniqueId(vendorIdFromUrl(seed.url), taken);
+    const host = (() => {
+      try {
+        return new URL(seed.url).hostname.replace(/^www\./, "");
+      } catch {
+        return seed.url;
+      }
+    })();
+    return {
+      id,
+      name: seed.name?.trim() || host,
+      urls: { home: seed.url, pricing: null, product: [] },
+    };
+  });
+}
+
+type ProposedClaim = {
+  capability: string;
+  icp: string;
+  pricingStructure: string;
+  definition: string;
+  terms?: string[];
+};
+
+type ProposedVendor = { seedUrl: string; name?: string; pricingUrl?: string | null };
+
+const TAXONOMY_SCHEMA = {
+  type: "object",
+  required: ["claims"],
+  properties: {
+    surfaceRule: {
+      type: "string",
+      description:
+        "One sentence stating how a reader judges LOUD vs QUIET vs ABSENT for this category (e.g. hero/top-nav = LOUD, deeper pages = QUIET, nowhere = ABSENT).",
+    },
+    claims: {
+      type: "array",
+      description:
+        "The distinct capability positions vendors in this category compete on. 8-16 of them. Only include claims you can actually see evidence for on the supplied pages.",
+      items: {
+        type: "object",
+        required: ["capability", "icp", "pricingStructure", "definition"],
+        properties: {
+          capability: {
+            type: "string",
+            description: "What is being claimed, precise enough to judge loud/quiet/absent. Max ~10 words.",
+          },
+          icp: { type: "string", description: "Which buyer/ICP this claim cell addresses (category vocabulary)." },
+          pricingStructure: {
+            type: "string",
+            description: "Which pricing structure the claim implies (e.g. per-seat, usage-based, flat, free-tier).",
+          },
+          definition: {
+            type: "string",
+            description:
+              "Operational definition a human (or classifier) uses to score any vendor's page LOUD/QUIET/ABSENT on this claim.",
+          },
+          terms: {
+            type: "array",
+            items: { type: "string" },
+            description: "Exact buyer phrasings for this claim, for deterministic mention matching. 2-5 terms.",
+          },
+        },
+      },
+    },
+    vendors: {
+      type: "array",
+      description: "Optional refinements: a clean display name per seed URL, and a pricing-page URL if one is clearly linked.",
+      items: {
+        type: "object",
+        required: ["seedUrl"],
+        properties: {
+          seedUrl: { type: "string" },
+          name: { type: "string" },
+          pricingUrl: { type: ["string", "null"] },
+        },
+      },
+    },
+  },
+} as const;
+
+function buildDossier(
+  vendors: MarketVendor[],
+  capture: ReturnType<typeof loadCaptureTexts>,
+  perVendorChars: number,
+): { dossier: string; unreadable: string[] } {
+  const { entries, textByHash } = capture;
+  const unreadable: string[] = [];
+  const blocks: string[] = [];
+  for (const vendor of vendors) {
+    const hash = entries.find((e) => e.vendorId === vendor.id && e.captureHash)?.captureHash ?? null;
+    const text = hash ? textByHash.get(hash) ?? "" : "";
+    if (!text.trim()) {
+      unreadable.push(vendor.id);
+      continue;
+    }
+    blocks.push(`### ${vendor.name} (${vendor.urls.home})\n${text.slice(0, perVendorChars)}`);
+  }
+  return { dossier: blocks.join("\n\n"), unreadable };
+}
+
+const INSTRUCTIONS = `You are seeding a competitive "market map" for a category. A market map breaks the category into CLAIMS — the distinct capability positions vendors compete on — so each (vendor x claim) cell can later be scored LOUD / QUIET / ABSENT from that vendor's pages.
+
+Propose the claim taxonomy for this category from the competitor homepages below. Rules:
+- Ground every claim in what is actually visible on the supplied pages. Do not invent positions no vendor mentions.
+- Each claim is a cell: a precise capability, the ICP it targets, and the pricing structure it implies.
+- Write each definition so a reader could judge ANY vendor's page LOUD/QUIET/ABSENT against it.
+- Aim for the 8-16 claims that genuinely differentiate vendors. Prefer specific, contested positions over generic table stakes.
+- Provide 2-5 verbatim buyer terms per claim for later mention matching.
+- Optionally return a cleaned display name and a pricing-page URL per seed vendor when evident.`;
+
+export async function suggestMarketConfig(options: SuggestTaxonomyOptions): Promise<SuggestTaxonomyResult> {
+  const { category } = options;
+  if (options.vendors.length === 0) throw new Error("suggestMarketConfig requires at least one seed vendor");
+  const maxClaims = options.maxClaims ?? DEFAULT_MAX_CLAIMS;
+  const perVendorChars = options.perVendorChars ?? DEFAULT_PER_VENDOR_CHARS;
+  const model = options.llm.model ?? DEFAULT_MODELS[options.llm.provider];
+
+  const vendors = provisionalVendors(options.vendors);
+  const anchorSeed = options.vendors.find((seed) => seed.anchor);
+  const anchorId = anchorSeed ? vendors[options.vendors.indexOf(anchorSeed)]?.id : undefined;
+
+  // Capture the seed homepages so the proposer only sees text we actually
+  // fetched (the SSRF guard in captureMarket applies to these user-supplied URLs).
+  await captureMarket(
+    { category, vendors, claims: [] },
+    { dir: options.capturesDir, runLabel: "bootstrap", fetchPage: options.fetchPage, now: options.now },
+  );
+  const capture = loadCaptureTexts(category, options.capturesDir);
+  const { dossier, unreadable } = buildDossier(vendors, capture, perVendorChars);
+  if (!dossier.trim()) {
+    throw new Error(
+      `market init --auto: none of the ${vendors.length} seed pages returned readable text — check the URLs are public homepages.`,
+    );
+  }
+
+  const prompt = `${INSTRUCTIONS}\n\nCategory: ${category}\n\nCompetitor homepages:\n${dossier}`;
+  const result = (await forcedToolCall(prompt, "propose_market_taxonomy", TAXONOMY_SCHEMA, model, options.llm)) as {
+    surfaceRule?: string;
+    claims?: ProposedClaim[];
+    vendors?: ProposedVendor[];
+  };
+
+  const takenClaimIds = new Set<string>();
+  const claims: MarketClaim[] = (result.claims ?? [])
+    .filter((claim) => claim?.capability && claim?.definition)
+    .slice(0, maxClaims)
+    .map((claim) => ({
+      id: uniqueId(slugify(claim.capability), takenClaimIds),
+      capability: claim.capability.trim(),
+      icp: (claim.icp ?? "").trim() || "general",
+      pricingStructure: (claim.pricingStructure ?? "").trim() || "unspecified",
+      definition: claim.definition.trim(),
+      ...(claim.terms?.length ? { terms: claim.terms.map((t) => t.trim()).filter(Boolean) } : {}),
+    }));
+
+  if (claims.length === 0) {
+    throw new Error("market init --auto: the model proposed no usable claims — try again or seed the taxonomy by hand.");
+  }
+
+  // Apply optional vendor refinements (display name + pricing URL), matched by seed URL.
+  const refinementByUrl = new Map((result.vendors ?? []).map((v) => [v.seedUrl, v]));
+  const refinedVendors: MarketVendor[] = vendors.map((vendor) => {
+    const refinement = refinementByUrl.get(vendor.urls.home);
+    const pricing =
+      refinement?.pricingUrl && /^https?:\/\//i.test(refinement.pricingUrl) ? refinement.pricingUrl : vendor.urls.pricing;
+    return {
+      ...vendor,
+      name: refinement?.name?.trim() || vendor.name,
+      urls: { ...vendor.urls, pricing },
+    };
+  });
+
+  const config: MarketConfig = {
+    category,
+    ...(anchorId ? { anchorVendor: anchorId } : {}),
+    vendors: refinedVendors,
+    claims,
+    surfaceRule:
+      result.surfaceRule?.trim() ||
+      "LOUD = hero copy OR top-level-nav named product with a dedicated page; QUIET = present on any indexed page below that; ABSENT = nowhere observed; UNOBSERVABLE = capture empty/failed — never score ABSENT from a failed capture.",
+  };
+
+  return { config, unreadableVendorIds: unreadable, model };
+}

package/src/planStore.ts

@@ -1,6 +1,7 @@
 import { chmodSync, mkdirSync, readdirSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { credentialsDir, ensureSecureHomeDir, writeSecureFile } from "./credentials.ts";
+import { computeApprovalDigests, loadOrCreateSigningKey } from "./integrity.ts";
 import type { ApprovalStatus, PatchPlan, PatchPlanRun } from "./types.ts";
 
 /**
@@ -16,6 +17,12 @@ export type StoredPlan = {
   status: ApprovalStatus;
   approvedOperationIds: string[];
   valueOverrides: Record<string, unknown>;
+  /**
+   * HMAC of each approved operation's content at approval time (see
+   * integrity.ts). Apply re-verifies these so a post-approval edit to the plan
+   * file is caught instead of written. Absent on plans approved before 0.26.0.
+   */
+  approvalDigests?: Record<string, string>;
   runs: PatchPlanRun[];
   createdAt: string;
   updatedAt: string;
@@ -125,13 +132,25 @@ export function createFilePlanStore(directory?: string): PlanStore {
           throw new Error(`Plan ${planId} has no operation ${operationId}.`);
         }
       }
+      const approvedOperationIds = Array.from(
+        new Set([...stored.approvedOperationIds, ...operationIds]),
+      );
+      const mergedOverrides = { ...stored.valueOverrides, ...valueOverrides };
+      // Bind the approval to the operation content so apply can detect a
+      // post-approval edit. Recompute over ALL approved ops (a later approve
+      // call may add overrides that change an earlier op's resolved value).
+      const approvalDigests = computeApprovalDigests(
+        stored.plan.operations,
+        approvedOperationIds,
+        mergedOverrides,
+        loadOrCreateSigningKey(),
+      );
       return write({
         ...stored,
         status: "approved",
-        approvedOperationIds: Array.from(
-          new Set([...stored.approvedOperationIds, ...operationIds]),
-        ),
-        valueOverrides: { ...stored.valueOverrides, ...valueOverrides },
+        approvedOperationIds,
+        valueOverrides: mergedOverrides,
+        approvalDigests,
       });
     },
 

package/src/schedule.ts

@@ -124,6 +124,12 @@ export function validateSchedulableArgv(argv: string[]): void {
           "plan store's approval state. Use `apply --plan-id <id>` and approve via `plans approve`.",
       );
     }
+    if (argv.includes("--value")) {
+      throw new Error(
+        "A scheduled apply cannot take --value — an unattended run must write exactly the values " +
+          "signed at approval. Set the value with `plans approve --value <op>=<v>` and re-approve.",
+      );
+    }
     return;
   }
   if (!Object.hasOwn(SCHEDULABLE, head)) {

package/src/types.ts

@@ -303,6 +303,22 @@ export type PatchOperation = {
    * member of the group.
    */
   groupId?: string;
+  /**
+   * Set only when a human explicitly chose to archive a record that shares an
+   * identity key with another (`bulk-update --archive --force-archive-duplicates`).
+   * Without it, apply refuses to archive_record a record the live snapshot still
+   * sees as a duplicate — archiving a duplicate discards data that merging keeps,
+   * and an agent on a dedupe task must not silently substitute archive for merge.
+   */
+  forceArchiveDuplicate?: boolean;
+  /**
+   * For irreversible operations (merge_records, archive_record): the field
+   * values of the records that will be destroyed, captured at plan-build time.
+   * Merges and archives cannot be undone on any provider, so this is the
+   * recovery artifact a human uses to recreate a record by hand if a merge or
+   * archive was wrong — the plan file IS the backup.
+   */
+  recoverySnapshot?: Record<string, unknown>[];
 };
 
 /**