@aionis/substrate 0.1.3 → 0.1.5

package/docs/ZVEC_SCALE_MAINTENANCE.md

@@ -0,0 +1,89 @@
+# Zvec Scale Maintenance Check
+
+This check validates the optional Zvec candidate sidecar under Substrate-scale write and maintenance operations.
+
+It proves a storage/index contract:
+
+- SQLite remains the truth store.
+- Zvec receives write-through node upserts while nodes are written.
+- `verify()` reports no missing, orphan, or stale entries after writes.
+- reopening the store rebuilds the sidecar from SQLite truth.
+- wide-window Zvec candidate search preserves canonical Substrate search ids.
+- narrow-window Zvec candidate search recovers seeded exact-node probes.
+- lifecycle transitions update the sidecar fingerprint.
+- checkpoint compaction and post-compaction reopen keep the sidecar verifiable.
+
+The check uses the same deterministic local text projection as the Runtime Zvec validation. It validates storage/index maintenance and candidate-window safety. It does not evaluate embedding-provider semantic quality.
+
+## Run
+
+```bash
+npm run check:zvec-scale -- \
+  --nodes 10000 \
+  --scopes 10 \
+  --relations 2000 \
+  --feedback 1000 \
+  --probes 100 \
+  --narrow-candidate-limit 20
+```
+
+The command writes a report under `reports/zvec-scale-*` unless `--output` is supplied.
+
+## Options
+
+- `--nodes <n>`: generated memory nodes.
+- `--scopes <n>`: generated scopes.
+- `--relations <n>`: generated relation rows.
+- `--feedback <n>`: generated feedback rows.
+- `--probes <n>`: exact-node search probes.
+- `--narrow-candidate-limit <n>`: Zvec candidate window for narrow seeded recovery probes.
+- `--transitions <n>`: lifecycle transitions to apply after reopen.
+- `--output <dir>`: report directory.
+- `--keep-store`: keep the temporary SQLite and Zvec files for inspection.
+
+## Report Interpretation
+
+Important fields:
+
+- `zvec_health.after_write`: write-through index health.
+- `zvec_health.after_reopen`: rebuild-on-open health.
+- `zvec_health.after_transitions`: lifecycle transition sync health.
+- `zvec_health.after_compact` and `after_compact_reopen`: compaction and reopen health.
+- `wide_parity_rate`: wide-window Zvec search matched canonical Substrate search.
+- `narrow_seed_hit_rate`: narrow-window Zvec search recovered seeded nodes.
+- `sqlite_bytes` and `zvec_bytes`: local storage footprint for the generated run.
+
+Any missing, orphan, stale, parity, or seeded-recovery failure means the Zvec sidecar needs maintenance work before it should be used for larger Substrate candidate preselection.
+
+## Local Validation Snapshot
+
+On 2026-06-26, the check was run locally with a 10k-node SQLite truth store and Zvec sidecar:
+
+```bash
+npm run check:zvec-scale -- \
+  --nodes 10000 \
+  --scopes 10 \
+  --relations 2000 \
+  --feedback 1000 \
+  --probes 100 \
+  --narrow-candidate-limit 20
+```
+
+Result:
+
+- nodes: 10,000
+- relations: 2,000
+- feedback records: 1,000
+- lifecycle transitions: 100
+- probes: 100
+- Zvec health after write/reopen/transitions/compact/compact-reopen: pass
+- wide candidate parity: 100%
+- narrow seeded recovery: 100%
+- SQLite bytes: 17,657,856
+- Zvec sidecar bytes: 19,528,279
+- write nodes with Zvec: 2,335 ms
+- close after write, including manifest flush: 171 ms
+- reopen and rebuild: 1,855 ms
+- post-compaction reopen and rebuild: 1,783 ms
+
+This validates the sidecar maintenance path at package scale: write-through indexing, rebuild, lifecycle-transition synchronization, compaction, and candidate-window safety all remain consistent with the SQLite truth store.

package/examples/live-sidecar/index.mjs

@@ -0,0 +1,189 @@
+import assert from "node:assert/strict";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import {
+  openSqliteAionisSubstrate,
+  runRuntimeLiveSidecarOnce,
+} from "../../dist/index.js";
+
+const scope = "repo-a";
+const workspace = await mkdtemp(join(tmpdir(), "aionis-substrate-live-sidecar-"));
+
+function createRuntimeLiteSource(path) {
+  const db = new DatabaseSync(path);
+  try {
+    db.exec(`
+      CREATE TABLE lite_memory_nodes (
+        id TEXT PRIMARY KEY,
+        scope TEXT NOT NULL,
+        client_id TEXT,
+        type TEXT NOT NULL,
+        tier TEXT NOT NULL,
+        title TEXT,
+        text_summary TEXT,
+        slots_json TEXT NOT NULL,
+        raw_ref TEXT,
+        evidence_ref TEXT,
+        embedding_vector_json TEXT,
+        embedding_model TEXT,
+        memory_lane TEXT NOT NULL,
+        producer_agent_id TEXT,
+        owner_agent_id TEXT,
+        owner_team_id TEXT,
+        embedding_status TEXT NOT NULL,
+        embedding_last_error TEXT,
+        salience REAL NOT NULL,
+        importance REAL NOT NULL,
+        confidence REAL NOT NULL,
+        redaction_version INTEGER NOT NULL,
+        commit_id TEXT NOT NULL,
+        created_at TEXT NOT NULL
+      );
+    `);
+  } finally {
+    db.close();
+  }
+}
+
+function insertRuntimeNode(path, row) {
+  const db = new DatabaseSync(path);
+  try {
+    db.prepare(`
+      INSERT INTO lite_memory_nodes (
+        id, scope, client_id, type, tier, title, text_summary, slots_json, raw_ref, evidence_ref,
+        embedding_vector_json, embedding_model, memory_lane, producer_agent_id, owner_agent_id,
+        owner_team_id, embedding_status, embedding_last_error, salience, importance, confidence,
+        redaction_version, commit_id, created_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      row.id,
+      scope,
+      `client-${row.id}`,
+      row.type,
+      row.tier,
+      row.title,
+      row.summary,
+      JSON.stringify(row.slots),
+      row.rawRef ?? null,
+      row.evidenceRef ?? null,
+      null,
+      "demo-embedding",
+      "execution",
+      "agent-a",
+      "agent-a",
+      "team-a",
+      "ready",
+      null,
+      0.8,
+      0.85,
+      row.confidence,
+      1,
+      "demo-commit",
+      row.createdAt,
+    );
+  } finally {
+    db.close();
+  }
+}
+
+try {
+  const runtimeSource = join(workspace, "runtime-lite.sqlite");
+  const substrateTarget = join(workspace, "substrate.sqlite");
+  const checkpoint = join(workspace, "runtime-live-checkpoint.json");
+  createRuntimeLiteSource(runtimeSource);
+
+  insertRuntimeNode(runtimeSource, {
+    id: "current-route",
+    type: "procedure",
+    tier: "hot",
+    title: "Current route",
+    summary: "Use src/runtime.ts after verifier passed.",
+    confidence: 0.95,
+    createdAt: "2026-06-26T00:00:00.000Z",
+    slots: {
+      summary_kind: "workflow_anchor",
+      contract_trust: "trusted",
+      target_files: ["src/runtime.ts", "tests/runtime.test.ts"],
+      execution_result_summary: { status: "passed" },
+    },
+  });
+
+  insertRuntimeNode(runtimeSource, {
+    id: "failed-branch",
+    type: "procedure",
+    tier: "hot",
+    title: "Failed branch",
+    summary: "The legacy src/legacy.ts path failed the verifier and should not steer the next turn.",
+    confidence: 0.9,
+    createdAt: "2026-06-26T00:01:00.000Z",
+    slots: {
+      summary_kind: "workflow_anchor",
+      contract_trust: "rejected",
+      target_files: ["src/legacy.ts"],
+      execution_result_summary: { status: "failed" },
+    },
+  });
+
+  insertRuntimeNode(runtimeSource, {
+    id: "raw-trace",
+    type: "trace_pointer",
+    tier: "cold",
+    title: "Raw terminal trace",
+    summary: "Full terminal trace is retained as payload evidence but should only be rehydrated on demand.",
+    rawRef: "file://trace.log",
+    confidence: 0.88,
+    createdAt: "2026-06-26T00:02:00.000Z",
+    slots: {
+      summary_kind: "raw_trace_pointer",
+      target_files: ["src/runtime.ts"],
+    },
+  });
+
+  const store = await openSqliteAionisSubstrate({ path: substrateTarget });
+  try {
+    const first = await runRuntimeLiveSidecarOnce({
+      sourcePath: runtimeSource,
+      target: store,
+      checkpointPath: checkpoint,
+      scope,
+    });
+    const second = await runRuntimeLiveSidecarOnce({
+      sourcePath: runtimeSource,
+      target: store,
+      checkpointPath: checkpoint,
+      scope,
+    });
+
+    const context = await store.previewContext({
+      scope,
+      query: "continue the current runtime implementation route",
+      maxPerBucket: 8,
+    });
+
+    assert.equal(first.apply_summary.nodes.applied, 3);
+    assert.equal(second.apply_summary.nodes.applied, 0);
+    assert.deepEqual(context.use_now.map((node) => node.id), ["current-route"]);
+    assert.deepEqual(context.do_not_use.map((node) => node.id), ["failed-branch"]);
+    assert.deepEqual(context.rehydrate.map((node) => node.id), ["raw-trace"]);
+
+    console.log(JSON.stringify({
+      ok: true,
+      runtime_source: runtimeSource,
+      substrate_target: substrateTarget,
+      first_sidecar_run: first.apply_summary.nodes,
+      second_sidecar_run: second.apply_summary.nodes,
+      governed_context: {
+        use_now: context.use_now.map((node) => node.id),
+        inspect_before_use: context.inspect_before_use.map((node) => node.id),
+        do_not_use: context.do_not_use.map((node) => node.id),
+        rehydrate: context.rehydrate.map((node) => node.id),
+      },
+    }, null, 2));
+  } finally {
+    await store.close();
+  }
+} finally {
+  await rm(workspace, { recursive: true, force: true });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aionis/substrate",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "Durable governed memory substrate for Aionis execution state.",
   "type": "module",
   "main": "./dist/index.js",
@@ -34,6 +34,10 @@
     "check:runtime-corpus": "node scripts/runtime-snapshot-corpus.ts",
     "check:runtime-reference-corpus": "node scripts/runtime-reference-corpus.ts",
     "check:runtime-sidecar": "node scripts/runtime-sidecar-check.ts",
+    "check:runtime-live-sidecar-soak": "node scripts/runtime-live-sidecar-soak.ts",
+    "check:runtime-product-bridge": "node scripts/runtime-product-bridge-gate.ts",
+    "check:runtime-zvec-index": "node scripts/runtime-zvec-candidate-index-check.ts",
+    "check:zvec-provider-embedding": "node scripts/zvec-provider-embedding-eval.ts",
     "make:runtime-product-reference": "node scripts/runtime-product-reference-fixture.ts",
     "check:external-admission-parity": "node scripts/external-admission-parity.ts",
     "check:runtime-dual-write": "node scripts/runtime-dual-write-experiment.ts",
@@ -42,8 +46,10 @@
     "check:registry-install": "node scripts/registry-install-smoke.ts",
     "check:published-runtime-smoke": "node scripts/published-runtime-smoke.ts",
     "check:scale": "node scripts/scale-test.ts",
+    "check:zvec-scale": "node scripts/zvec-scale-test.ts",
     "example:basic": "npm run build && node examples/basic/index.mjs",
-    "check:release": "npm run typecheck && npm test && npm run bench:contract && npm run example:basic && npm run check:pack && npm run check:install-smoke"
+    "example:live-sidecar": "npm run build && node examples/live-sidecar/index.mjs",
+    "check:release": "npm run typecheck && npm test && npm run bench:contract && npm run example:basic && npm run check:runtime-live-sidecar-soak && npm run check:pack && npm run check:install-smoke"
   },
   "repository": {
     "type": "git",
@@ -66,6 +72,15 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@types/node": "^26.0.1",
+    "@zvec/zvec": "^0.5.0",
     "typescript": "^6.0.3"
+  },
+  "peerDependencies": {
+    "@zvec/zvec": "^0.5.0"
+  },
+  "peerDependenciesMeta": {
+    "@zvec/zvec": {
+      "optional": true
+    }
   }
 }