@hassangameryt/freeflow 0.2.0 → 0.3.0

package/{plugins/freeflow/router/dist → router/dist/vault}/vault.js

@@ -2,7 +2,8 @@ import { createHash } from "node:crypto";
 import { mkdir, open, readFile, rename, unlink, writeFile } from "node:fs/promises";
 import { homedir } from "node:os";
 import { dirname, join, resolve } from "node:path";
-import { DEFAULT_VAULT_RETENTION, DEFAULT_VAULT_ROOT } from "./config.js";
+import { DEFAULT_VAULT_RETENTION, DEFAULT_VAULT_ROOT } from "../config/config.js";
+import { createLocalVaultIndex, recordVaultIndexFailure } from "./vault-index.js";
 const sessionIndexWriteLocks = new Map();
 const SESSION_INDEX_LOCK_RETRY_MS = 5;
 const SESSION_INDEX_LOCK_TIMEOUT_MS = 5_000;
@@ -79,6 +80,7 @@ export async function findExactDuplicateTextOutput(vault, options) {
     return undefined;
 }
 export async function storeCommandOutput(vault, options) {
+    assertStorablePersistence(options.persistence);
     const createdAt = options.createdAt ?? new Date().toISOString();
     const combined = options.combined ?? combineOutputSections(options.stdout, options.stderr);
     const decisionIds = options.decisionIds ?? [];
@@ -106,6 +108,7 @@ export async function storeCommandOutput(vault, options) {
     });
     const objectId = objectIdFromHash(payloadHash);
     const outputId = outputIdFromHash(payloadHash);
+    const recordId = recordIdFromHash(payloadHash);
     const objectDir = objectDirectory(vault, objectId);
     const paths = {
         meta: join(objectDir, "meta.json"),
@@ -116,6 +119,7 @@ export async function storeCommandOutput(vault, options) {
     const record = {
         kind: "command",
         outputId,
+        recordId,
         objectId,
         command: options.command,
         createdAt,
@@ -139,9 +143,14 @@ export async function storeCommandOutput(vault, options) {
         },
         fingerprints,
         decisionIds,
+        producer: options.producer ?? { kind: "command" },
+        persistence: options.persistence ?? exactVaultPersistence(outputId),
         contentHashSha256: payloadHash,
         retention: vault.retention,
     };
+    if (options.lineage !== undefined) {
+        record.lineage = options.lineage;
+    }
     const commandExpiresAt = expiresAt(createdAt, vault.retention);
     if (commandExpiresAt !== undefined) {
         record.expiresAt = commandExpiresAt;
@@ -158,9 +167,11 @@ export async function storeCommandOutput(vault, options) {
     await writeFile(paths.combined, combined, "utf8");
     await writeJson(paths.meta, record);
     await addRecordToSessionIndex(vault, options.sessionId, record);
+    await indexVaultRecordAfterAppend(vault, options.sessionId, record, combined, "combined");
     return record;
 }
 export async function storeTextOutput(vault, options) {
+    assertStorablePersistence(options.persistence);
     const createdAt = options.createdAt ?? new Date().toISOString();
     const decisionIds = options.decisionIds ?? [];
     const fingerprints = textOutputFingerprints({ raw: options.raw });
@@ -173,6 +184,7 @@ export async function storeTextOutput(vault, options) {
     });
     const objectId = objectIdFromHash(payloadHash);
     const outputId = outputIdFromHash(payloadHash);
+    const recordId = recordIdFromHash(payloadHash);
     const objectDir = objectDirectory(vault, objectId);
     const paths = {
         meta: join(objectDir, "meta.json"),
@@ -181,6 +193,7 @@ export async function storeTextOutput(vault, options) {
     const record = {
         kind: "text",
         outputId,
+        recordId,
         objectId,
         sourceKind: options.sourceKind,
         createdAt,
@@ -190,9 +203,14 @@ export async function storeTextOutput(vault, options) {
         hashes: { rawSha256: sha256Text(options.raw) },
         fingerprints,
         decisionIds,
+        producer: options.producer ?? producerForTextSourceKind(options.sourceKind),
+        persistence: options.persistence ?? exactVaultPersistence(outputId),
         contentHashSha256: payloadHash,
         retention: vault.retention,
     };
+    if (options.lineage !== undefined) {
+        record.lineage = options.lineage;
+    }
     const textExpiresAt = expiresAt(createdAt, vault.retention);
     if (textExpiresAt !== undefined) {
         record.expiresAt = textExpiresAt;
@@ -201,9 +219,65 @@ export async function storeTextOutput(vault, options) {
     await writeFile(paths.raw, options.raw, "utf8");
     await writeJson(paths.meta, record);
     await addRecordToSessionIndex(vault, options.sessionId, record);
+    await indexVaultRecordAfterAppend(vault, options.sessionId, record, options.raw, "raw");
+    return record;
+}
+export async function storeMetadataOutput(vault, options) {
+    assertStorablePersistence(options.persistence);
+    const createdAt = options.createdAt ?? new Date().toISOString();
+    const decisionIds = options.decisionIds ?? [];
+    const payloadHash = sha256Json({
+        kind: "metadata",
+        createdAt,
+        sourceKind: options.sourceKind,
+        rawLineCount: options.rawLineCount,
+        rawByteCount: options.rawByteCount,
+        rawSha256: options.rawSha256,
+        metadata: options.metadata,
+        decisionIds,
+    });
+    const objectId = objectIdFromHash(payloadHash);
+    const outputId = outputIdFromHash(payloadHash);
+    const recordId = recordIdFromHash(payloadHash);
+    const objectDir = objectDirectory(vault, objectId);
+    const paths = {
+        meta: join(objectDir, "meta.json"),
+    };
+    const record = {
+        kind: "metadata",
+        outputId,
+        recordId,
+        objectId,
+        sourceKind: options.sourceKind,
+        createdAt,
+        paths,
+        lineCounts: { raw: options.rawLineCount },
+        byteCounts: { raw: options.rawByteCount },
+        hashes: options.rawSha256 !== undefined ? { rawSha256: options.rawSha256 } : {},
+        decisionIds,
+        producer: options.producer ?? producerForTextSourceKind(options.sourceKind),
+        persistence: options.persistence ?? metadataOnlyPersistence(outputId),
+        contentHashSha256: payloadHash,
+        retention: vault.retention,
+    };
+    if (options.metadata !== undefined) {
+        record.metadata = options.metadata;
+    }
+    if (options.lineage !== undefined) {
+        record.lineage = options.lineage;
+    }
+    const metadataExpiresAt = expiresAt(createdAt, vault.retention);
+    if (metadataExpiresAt !== undefined) {
+        record.expiresAt = metadataExpiresAt;
+    }
+    await mkdir(objectDir, { recursive: true });
+    await writeJson(paths.meta, record);
+    await addRecordToSessionIndex(vault, options.sessionId, record);
+    await indexVaultRecordAfterAppend(vault, options.sessionId, record, undefined);
     return record;
 }
 export async function storeRepoFileReference(vault, options) {
+    assertStorablePersistence(options.persistence);
     const createdAt = options.createdAt ?? new Date().toISOString();
     const decisionIds = options.decisionIds ?? [];
     const payloadHash = sha256Json({
@@ -215,6 +289,7 @@ export async function storeRepoFileReference(vault, options) {
     });
     const objectId = objectIdFromHash(payloadHash);
     const outputId = outputIdFromHash(payloadHash);
+    const recordId = recordIdFromHash(payloadHash);
     const objectDir = objectDirectory(vault, objectId);
     const paths = {
         meta: join(objectDir, "meta.json"),
@@ -222,14 +297,20 @@ export async function storeRepoFileReference(vault, options) {
     const record = {
         kind: "repo-file",
         outputId,
+        recordId,
         objectId,
         path: options.path,
         paths,
         decisionIds,
         createdAt,
+        producer: options.producer ?? { kind: "repo" },
+        persistence: options.persistence ?? metadataOnlyPersistence(),
         contentHashSha256: payloadHash,
         retention: vault.retention,
     };
+    if (options.lineage !== undefined) {
+        record.lineage = options.lineage;
+    }
     const repoExpiresAt = expiresAt(createdAt, vault.retention);
     if (repoExpiresAt !== undefined) {
         record.expiresAt = repoExpiresAt;
@@ -240,6 +321,7 @@ export async function storeRepoFileReference(vault, options) {
     await mkdir(objectDir, { recursive: true });
     await writeJson(paths.meta, record);
     await addRecordToSessionIndex(vault, options.sessionId, record);
+    await indexVaultRecordAfterAppend(vault, options.sessionId, record, undefined);
     return record;
 }
 export async function readVaultRecord(vault, sessionId, outputId) {
@@ -279,16 +361,40 @@ export async function readSessionIndex(vault, sessionId) {
         return emptySessionIndex(sessionId, now);
     }
 }
+function assertStorablePersistence(persistence) {
+    if (persistence?.recoverability === "none") {
+        throw new Error("Cannot store output with persistence recoverability none; skip vault storage instead.");
+    }
+}
+async function indexVaultRecordAfterAppend(vault, sessionId, record, text, stream) {
+    try {
+        const index = createLocalVaultIndex(vault);
+        const metadata = {
+            sessionId,
+            ...(stream !== undefined ? { stream } : {}),
+        };
+        await index.indexRecord(record, text, metadata);
+    }
+    catch (error) {
+        await recordVaultIndexFailure(vault, error).catch(() => undefined);
+    }
+}
 async function addRecordToSessionIndex(vault, sessionId, record) {
     await withSessionIndexWriteLock(vault, sessionId, async () => {
         const index = await readSessionIndex(vault, sessionId);
         const updatedAt = new Date().toISOString();
         const entry = {
             outputId: record.outputId,
+            recordId: record.recordId,
             objectId: record.objectId,
             kind: record.kind,
             createdAt: record.createdAt,
+            producer: record.producer,
+            persistence: record.persistence,
         };
+        if (record.lineage !== undefined) {
+            entry.lineage = record.lineage;
+        }
         if (record.kind === "command") {
             entry.executionStatus = record.executionStatus;
         }
@@ -394,6 +500,9 @@ function streamPath(record, stream) {
         }
         return record.paths.raw;
     }
+    if (record.kind === "metadata") {
+        throw new Error("Metadata only records store no raw stream.");
+    }
     throw new Error("Repo file reference records store metadata only and have no raw stream.");
 }
 function sessionIndexPath(vault, sessionId) {
@@ -411,6 +520,48 @@ function objectIdFromHash(hash) {
 function outputIdFromHash(hash) {
     return `ffout_${hash.slice(0, 24)}`;
 }
+function recordIdFromHash(hash) {
+    return `ffrec_${hash.slice(0, 24)}`;
+}
+function exactVaultPersistence(outputId) {
+    return {
+        status: "vaulted",
+        recoverability: "exact",
+        recoveryOutputId: outputId,
+        outputId,
+    };
+}
+function metadataOnlyPersistence(outputId) {
+    return {
+        status: "metadata_only",
+        recoverability: "metadata_only",
+        ...(outputId !== undefined ? { outputId } : {}),
+    };
+}
+function producerForTextSourceKind(sourceKind) {
+    if (sourceKind === "native") {
+        return { kind: "native" };
+    }
+    if (sourceKind === "script") {
+        return { kind: "script" };
+    }
+    if (sourceKind === "mcp") {
+        return { kind: "mcp" };
+    }
+    if (sourceKind === "web") {
+        return { kind: "web" };
+    }
+    if (sourceKind === "fetch") {
+        return { kind: "fetch" };
+    }
+    if (sourceKind === "code_search") {
+        return { kind: "code_search" };
+    }
+    if (sourceKind === "transform") {
+        return { kind: "transform" };
+    }
+    return { kind: "other" };
+}
 function combineOutputSections(stdout, stderr) {
     if (stdout.length === 0) {
         return stderr;

package/skills/design-for-depth/SKILL.md

@@ -0,0 +1,90 @@
+---
+name: design-for-depth
+description: Use when shaping, reviewing, implementing, or diagnosing work where module/interface/seam choices affect complexity, locality, testability, future change, or repeated edge-case churn. Use for architecture direction, growing scope, shallow modules, bad seams, broad refactors, and review loops that keep finding new edge cases.
+---
+
+# Design For Depth
+
+Use this as a lens, not a phase.
+
+Core rule: if complexity is spreading across callers, tests, docs, artifacts, or review comments, stop and classify the design pressure before patching forward.
+
+Goal: hide useful complexity behind stable interfaces so future callers, tests, reviewers, and agents coordinate less.
+
+Do not add architecture ceremony when the next action is small, local, and reversible. Do not use architecture language to hide product or policy decisions.
+
+## Load When Needed
+
+Read `references/software-design-philosophy.md` when shaping architecture direction, reviewing a design-heavy artifact, or diagnosing repeated design failure.
+
+Read `references/design-pressure-signals.md` when reviewing code/work for shallow modules, scattered policy, edge-case churn, or refactor candidates.
+
+Do not load references for every small design question. The active skill should be enough for routine routing.
+
+## Language
+
+Use these terms consistently:
+
+- **Module** — anything with an interface and implementation: function, class, package, subsystem, workflow slice.
+- **Interface** — everything a caller, user, test, or future agent must know: types, invariants, ordering, errors, configuration, performance, side effects, and policy context.
+- **Implementation** — details hidden behind the interface.
+- **Depth** — leverage at the interface. Deep modules hide useful behavior behind a smaller interface.
+- **Shallow module** — interface is nearly as complex as implementation, or callers coordinate details the module should own.
+- **Seam** — where behavior, ownership, or variation can change without editing surrounding work.
+- **Adapter** — a concrete thing satisfying an interface at a seam.
+- **Locality** — changes, bugs, decisions, and verification stay near one module.
+- **Leverage** — future work gets more behavior from less interface knowledge.
+
+Avoid using “architecture” as a vague quality claim. Name the module, interface, hidden decision, seam, adapter, locality, or leverage.
+
+## Signals
+
+Use this lens when evidence shows:
+
+- one concept requires many unrelated file, test, doc, or reviewer-comment changes;
+- callers know ordering, flags, retries, cleanup, cache, auth, billing, migration, privacy, permissions, or compatibility policy;
+- tests reach past the interface to assert normal behavior;
+- review findings become a stream of edge-case patches;
+- the same conditional or policy appears in multiple callers;
+- a module is mostly pass-through naming around another module;
+- a seam exists only for theoretical future variation;
+- a spec or plan encodes detailed implementation guesses before evidence exists;
+- the agent can only explain correctness by listing coordinated steps across modules.
+
+Do not use this lens just because work is “important.” Use it when interface shape changes the next action.
+
+## Tests
+
+Use the smallest test that changes the route:
+
+- **Deletion test:** if deleting the module makes complexity vanish, it may be pass-through ceremony. If complexity reappears across callers, tests, docs, or review comments, it was earning its keep.
+- **Interface test surface:** callers and tests should exercise normal behavior through the same interface. If tests bypass it, mock internals, or duplicate setup, the module shape may be wrong.
+- **Variation test:** one adapter is a hypothetical seam. Two adapters, a real test double, or a known upcoming variation justify a seam better.
+- **Locality test:** one behavior change should not scatter policy, error handling, retries, logging, auth, billing, compatibility, docs, and tests across unrelated places.
+- **Decision-hiding test:** design decisions likely to change should live behind modules, not in every caller or artifact.
+- **Obscurity test:** if a fresh agent must read many files to learn a simple rule, the design may be obscure even if each file is small.
+
+## Route
+
+If the current direction spreads complexity, do not patch forward silently.
+
+Classify the pressure:
+
+- **Local fix:** same interface, complexity stays hidden; continue in the current phase.
+- **Plan defect:** slice boundary, check, or implementation path is wrong; revise the plan.
+- **Spec gap:** behavior, scope, or acceptance is unclear; revise the spec or return to discovery.
+- **Owner decision:** public API, compatibility, security, privacy, billing, data-loss, permissions, migration, or hard-to-reverse architecture changes; use the interview gate.
+- **Refactor candidate:** current behavior is valid but the module is shallow; propose a bounded deepening path before editing.
+- **Stop/defer:** design pressure is real but not worth solving in this scope; record or defer it explicitly.
+
+## Non-Goals
+
+Do not:
+
+- introduce abstractions for imagined future variation;
+- hide product or policy decisions inside architecture wording;
+- rewrite source truth to make a design cleaner;
+- turn every feature into upfront architecture design;
+- broaden a narrow fix into a refactor without user approval;
+- treat line count, file count, indirection, or pattern names as proof of quality;
+- claim a design is better without evidence from code, tests, review, usage, or an explicit owner decision.

package/skills/design-for-depth/references/design-pressure-signals.md

@@ -0,0 +1,265 @@
+# Design Pressure Signals
+
+Use this reference when reviewing code, artifacts, or work for shallow modules and complexity spread.
+
+The goal is not to find every smell. The goal is to notice pressure that changes the next route.
+
+## Quick Classifier
+
+Ask:
+
+1. What is the module?
+2. What is the interface?
+3. What details are callers forced to know?
+4. What decision is likely to change?
+5. Where would that decision be localized?
+6. Would solving this now fit the approved scope?
+
+Then classify:
+
+- **Continue:** no design pressure that changes the next action.
+- **Local fix:** improve within current module/interface.
+- **Plan revision:** slice boundary or verification path is wrong.
+- **Spec/discovery:** behavior, scope, or acceptance is unclear.
+- **Owner decision:** public API, compatibility, security, privacy, billing, data loss, permissions, migration, or hard-to-reverse architecture.
+- **Deferred deepening:** real design pressure, but not worth solving in this scope.
+
+## Signals And What They Usually Mean
+
+### Shotgun Surgery
+
+Signal: one concept requires many unrelated edits.
+
+Examples:
+
+- retry policy added to every route;
+- permission check copied into API, UI, and job worker;
+- billing grace-period logic repeated in webhook, email, and dashboard;
+- cache freshness flags threaded through callers.
+
+Likely issue: a likely-changing decision is not hidden behind a module.
+
+Route:
+
+- If behavior/scope is unsettled: discovery or spec.
+- If behavior is settled but slice spreads edits: plan revision or refactor candidate.
+
+### Caller Choreography
+
+Signal: callers must perform steps in the right order.
+
+Examples:
+
+```text
+open connection -> set retry -> register cleanup -> call provider -> translate error -> log metric
+```
+
+Likely issue: interface exposes implementation sequence.
+
+Better interface asks for outcome:
+
+```text
+deliverNotification(invitation)
+```
+
+while the module owns retry, fallback, logging, cleanup, and provider errors.
+
+### Scattered Policy
+
+Signal: product or operational rules appear in many places.
+
+Examples:
+
+- billing downgrade timing in UI, webhook, worker, and tests;
+- notification fallback rules in every route;
+- authorization rules in frontend and backend separately;
+- migration compatibility gates inside individual callers.
+
+Likely issue: policy is not localized.
+
+Stop if changing it would alter product, security, privacy, billing, permissions, public API, compatibility, or data-loss behavior.
+
+### Test Knows Too Much
+
+Signal: tests assert implementation sequencing instead of behavior.
+
+Examples:
+
+- tests assert `sendEmail` then `logFailure` then `sendSms` in every route;
+- tests mock five helpers to verify one user-visible behavior;
+- tests duplicate retry counts and provider error types;
+- tests need private methods or internal state.
+
+Likely issue: the production interface is not the right test seam.
+
+Do not automatically add mocks. Ask whether the module interface should own the behavior.
+
+### Edge-Case Patch Stream
+
+Signal: each review pass finds another special case.
+
+Examples:
+
+- “also handle null phone”; then “also handle email bounce”; then “also handle duplicate invite”; then “also handle retry telemetry.”
+
+Likely issue: review is exposing missing behavior ownership, not isolated bugs.
+
+Route:
+
+- classify findings;
+- stop at review cap;
+- diagnose whether discovery, spec, plan, source truth, module shape, or reviewer context is wrong.
+
+### Pass-Through Wrapper
+
+Signal: a module mostly renames another call.
+
+Example:
+
+```ts
+function sendTeamInviteEmail(email, subject, options) {
+  return sendEmail(email, subject, options);
+}
+```
+
+Likely issue: shallow wrapper.
+
+Options:
+
+- delete it if it hides nothing;
+- deepen it by moving real policy behind it;
+- keep it only if it expresses a stable domain interface that will soon hide behavior.
+
+### Leaky Interface
+
+Signal: callers know provider/database/cache internals.
+
+Examples:
+
+- callers catch Stripe-specific errors;
+- UI code knows database enum transitions;
+- API callers pass cache invalidation flags;
+- tests assert internal queue names.
+
+Likely issue: implementation detail leaked through interface.
+
+Route to design if leakage blocks safe change or verification.
+
+### God Module
+
+Signal: one file owns unrelated responsibilities.
+
+Examples:
+
+- route file validates input, applies policy, calls providers, logs metrics, and formats UI copy;
+- workflow module owns routing, vaulting, parsing, rendering, and config mutation.
+
+Likely issue: missing internal modules or seams.
+
+Do not split by file type alone. Split around hidden decisions and interfaces.
+
+### Speculative Seam
+
+Signal: interface/adapters/factory exist for only imagined future variation.
+
+Examples:
+
+- `PaymentProviderAdapterFactoryRegistry` when only Stripe exists and no migration is planned;
+- strategies for every small formatting function;
+- generic repository around a single simple query.
+
+Likely issue: indirection without leverage.
+
+Use the variation test: one adapter is hypothetical; two adapters or known upcoming variation justify a seam better.
+
+### Premature Artifact Detail
+
+Signal: spec or plan chooses exact classes, factories, tables, or algorithms before evidence exists.
+
+Likely issue: artifact is hiding uncertainty as implementation detail.
+
+Better:
+
+```text
+Tentative: likely need a policy-owning module if retry/fallback rules spread.
+Open: whether provider variation is real in this scope.
+Stop: revisit plan if slice requires touching every caller.
+```
+
+## Before / After Examples
+
+### Notification Retry
+
+Bad direction:
+
+```text
+Add retryCount to each route that sends notifications.
+```
+
+Pressure:
+
+- callers know retry policy;
+- fallback and telemetry are duplicated;
+- tests assert provider sequencing;
+- adding another provider touches every caller.
+
+Better direction:
+
+```text
+Introduce a notification delivery module whose interface accepts the notification intent and owns retry, fallback, provider errors, and telemetry.
+```
+
+Still ask owner decisions before changing user-visible delivery semantics.
+
+### Billing Grace Period
+
+Bad direction:
+
+```text
+Patch webhook to downgrade immediately, update email text, adjust dashboard copy.
+```
+
+Pressure:
+
+- billing policy spread across systems;
+- likely source-truth conflict with docs/tests;
+- user-owned billing behavior.
+
+Better direction:
+
+```text
+Stop at interview gate. Decide billing policy first. Then localize policy behind a billing-state transition module if implementation spreads.
+```
+
+### Cache Freshness
+
+Bad direction:
+
+```text
+Thread forceRefresh through every caller.
+```
+
+Pressure:
+
+- callers know cache invalidation policy;
+- tests must set timing flags;
+- source truth may allow caching.
+
+Better direction:
+
+```text
+Discover expected freshness boundary. If source truth supports it, hide invalidation/freshness rules behind a cache-aware read interface.
+```
+
+## False Positives
+
+Do not overreact when:
+
+- a small repeated pattern is stable and local;
+- a one-off script does not need long-term depth;
+- a wrapper exists for public compatibility;
+- broad edits are mechanical and source-backed;
+- tests inspect internals only for an internal module whose interface is intentionally internal;
+- deeper design would expand scope beyond the user's goal.
+
+When in doubt, classify as deferred design pressure rather than silently refactoring.