@zaip-ai/cli 2.2.0 → 3.0.0

package/templates/.agents/dashboard/packages/local-runtime/src/sqlite-read-model-rebuilds-disposable-snapshot-serves-zaip-records.test.ts

@@ -0,0 +1,276 @@
+import { mkdir, mkdtemp, readFile, utimes, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { rebuildSqliteReadModel, SqliteReadModelFileSystem } from "./sqlite-read-model.js";
+
+const roots: string[] = [];
+const sqliteTestTempRoot = join(
+  resolve(import.meta.dirname, "../../../../../.."),
+  ".tmp",
+  "sqlite-read-model-tests",
+);
+async function createSqliteTestRoot(): Promise<string> {
+  await mkdir(sqliteTestTempRoot, { recursive: true });
+  return mkdtemp(join(sqliteTestTempRoot, "fixture-"));
+}
+afterEach(async () => {
+  vi.useRealTimers();
+  vi.restoreAllMocks();
+  await Promise.all(
+    roots
+      .splice(0)
+      .map((root) =>
+        import("node:fs/promises").then(({ rm }) => rm(root, { recursive: true, force: true })),
+      ),
+  );
+});
+describe("SQLite dashboard read model", () => {
+  it("rebuilds a disposable snapshot and serves ZAIP records from SQLite", async () => {
+    const root = await createSqliteTestRoot();
+    roots.push(root);
+    await mkdir(join(root, ".agents", "records", "tasks"), { recursive: true });
+    await mkdir(join(root, ".agents", "config"), { recursive: true });
+    await writeFile(join(root, ".agents", "records", "tasks", "task.md"), "# task\n", "utf8");
+    await writeFile(
+      join(root, ".agents", "config", "project.yaml"),
+      "zaip:\n  enabled: true\n",
+      "utf8",
+    );
+    const result = await rebuildSqliteReadModel(root);
+    const filesystem = new SqliteReadModelFileSystem(root);
+    expect(result.files).toBe(2);
+    await expect(filesystem.readText(".agents/records/tasks/task.md")).resolves.toContain("task");
+    await expect(filesystem.listFiles(".agents/records", ".md")).resolves.toEqual([
+      ".agents/records/tasks/task.md",
+    ]);
+    filesystem.close();
+  });
+  it("serves canonical Task trace inputs without lifecycle manifests", async () => {
+    const root = await createSqliteTestRoot();
+    roots.push(root);
+    await mkdir(join(root, ".agents", "records", "tasks", "2026", "08"), {
+      recursive: true,
+    });
+    await mkdir(join(root, ".agents", "config"), { recursive: true });
+    await writeFile(
+      join(root, ".agents", "config", "project.yaml"),
+      "zaip:\n  enabled: true\n",
+      "utf8",
+    );
+    await writeFile(
+      join(root, ".agents", "records", "tasks", "2026", "08", "TASK-TRACE.md"),
+      "---\nid: TASK-TRACE\ntitle: Canonical trace\nstatus: completed\nrisk: standard\ncreated_at: 2026-08-01T12:00:00.000Z\n---\n# Canonical trace\n",
+      "utf8",
+    );
+    await rebuildSqliteReadModel(root);
+    const filesystem = new SqliteReadModelFileSystem(root);
+    expect(filesystem.canonicalTraceFiles(["TASK-TRACE"])).toMatchObject([
+      { path: ".agents/records/tasks/2026/08/TASK-TRACE.md" },
+    ]);
+    expect(filesystem.taskTraceUsage("TASK-TRACE")).toEqual({
+      run_count: 0,
+      duration_ms: { value: null, known_runs: 0, unknown_runs: 0, coverage_ratio: 1 },
+      total_tokens: { value: null, known_runs: 0, unknown_runs: 0, coverage_ratio: 1 },
+      cost: {
+        value: null,
+        known_runs: 0,
+        unknown_runs: 0,
+        coverage_ratio: 1,
+        currency: null,
+      },
+    });
+    filesystem.close();
+  });
+  it("invalidates a nested canonical source change and ignores legacy aggregate telemetry", async () => {
+    const root = await createSqliteTestRoot();
+    roots.push(root);
+    const nested = join(root, ".agents", "records", "tasks", "nested", "task.md");
+    await mkdir(join(root, ".agents", "config"), { recursive: true });
+    await mkdir(join(root, ".agents", "records", "tasks", "nested"), { recursive: true });
+    await writeFile(
+      join(root, ".agents", "config", "project.yaml"),
+      "zaip:\n  enabled: true\n",
+      "utf8",
+    );
+    await writeFile(nested, "# before\n", "utf8");
+    await rebuildSqliteReadModel(root);
+    await writeFile(nested, "# after\n", "utf8");
+    const changedAt = new Date(Date.now() + 2_000);
+    await utimes(nested, changedAt, changedAt);
+    await rebuildSqliteReadModel(root);
+    const updated = new SqliteReadModelFileSystem(root);
+    await expect(updated.readText(".agents/records/tasks/nested/task.md")).resolves.toContain(
+      "after",
+    );
+    updated.close();
+
+    const duplicate = join(
+      root,
+      ".agents",
+      "records",
+      "telemetry",
+      "2026",
+      "08",
+      "writers",
+      "w",
+      "chunks",
+      "c.jsonl",
+    );
+    await mkdir(
+      join(root, ".agents", "records", "telemetry", "2026", "08", "writers", "w", "chunks"),
+      { recursive: true },
+    );
+    await writeFile(
+      join(root, ".agents", "records", "telemetry", "2026", "08", "writers", "w", "manifest.json"),
+      JSON.stringify({
+        chunks: [{ path: ".agents/records/telemetry/2026/08/writers/w/chunks/c.jsonl" }],
+      }),
+      "utf8",
+    );
+    await writeFile(
+      duplicate,
+      `${JSON.stringify({ agent_invocation_id: "same", summary_hash: "one" })}\n${JSON.stringify({ agent_invocation_id: "same", summary_hash: "two" })}\n`,
+      "utf8",
+    );
+    await expect(rebuildSqliteReadModel(root)).resolves.toMatchObject({ files: 2 });
+    expect(await readFile(nested, "utf8")).toContain("after");
+    const retained = new SqliteReadModelFileSystem(root);
+    await expect(retained.readText(".agents/records/tasks/nested/task.md")).resolves.toContain(
+      "after",
+    );
+    retained.close();
+  });
+  it("does not promote legacy aggregate telemetry into canonical activity facts", async () => {
+    const root = await createSqliteTestRoot();
+    roots.push(root);
+    await mkdir(
+      join(root, ".agents", "records", "telemetry", "2026", "08", "writers", "w", "chunks"),
+      { recursive: true },
+    );
+    await mkdir(join(root, ".agents", "config"), { recursive: true });
+    await writeFile(
+      join(root, ".agents", "config", "project.yaml"),
+      "zaip:\n  enabled: true\n",
+      "utf8",
+    );
+    const summary = {
+      agent_invocation_id: "root",
+      summary_hash: "hash",
+      root_agent_invocation_id: "root",
+      parent_agent_invocation_id: null,
+      execution_id: null,
+      task_id: null,
+      spec_id: null,
+      prd_id: null,
+      change_id: null,
+      run_id: "RUN-1",
+      occurred_at: "2026-08-01T12:00:00.000Z",
+      invocation_kind: "orchestrator",
+      delegation_depth: 0,
+      source_event_count: 1,
+      historical_cost: {
+        estimated_usd: 2,
+        estimated_brl: 10,
+        fx_rate: 5,
+        pricing_model: "model",
+        pricing_source: "canonical",
+      },
+      quantitative_metrics: { duration_ms: { known_sum: 5 }, tokens_total: { known_sum: 0 } },
+      usage_segments: [
+        {
+          provider: "openai",
+          model: "model",
+          service_tier: "default",
+          reasoning_effort: "medium",
+          tokens: { input: 0, cached_input: 0, output: 0, reasoning: 0, total: 0 },
+          historical_cost: {
+            estimated_usd: 2,
+            estimated_brl: 10,
+            fx_rate: 5,
+            pricing_model: "model",
+            pricing_source: "canonical",
+          },
+        },
+      ],
+    };
+    await writeFile(
+      join(
+        root,
+        ".agents",
+        "records",
+        "telemetry",
+        "2026",
+        "08",
+        "writers",
+        "w",
+        "chunks",
+        "c.jsonl",
+      ),
+      `${JSON.stringify(summary)}\n`,
+      "utf8",
+    );
+    await writeFile(
+      join(root, ".agents", "records", "telemetry", "2026", "08", "writers", "w", "manifest.json"),
+      JSON.stringify({
+        chunks: [{ path: ".agents/records/telemetry/2026/08/writers/w/chunks/c.jsonl" }],
+      }),
+      "utf8",
+    );
+    await rebuildSqliteReadModel(root);
+    const filesystem = new SqliteReadModelFileSystem(root);
+    expect(filesystem.firstFoldAggregate({ from: "2026-08-01", to: "2026-08-01" })).toEqual({
+      days: 0,
+      total_run_count: 0,
+      agent_invocation_count: 0,
+      llm_call_count: null,
+      duration_ms: null,
+      total_tokens: null,
+      cost_brl: null,
+      execution_day_count: 0,
+    });
+    const database = new DatabaseSync(resolve(root, ".local/sqlite/zaip-aios-dashboard.sqlite"), {
+      readOnly: true,
+    });
+    const columns = (table: string) =>
+      (database.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map(
+        (row) => row.name,
+      );
+    expect(columns("zaip_vw_daily_aggregates")).toEqual(
+      expect.arrayContaining([
+        "day",
+        "invocation_count",
+        "total_duration",
+        "total_tokens",
+        "total_cost_usd",
+        "total_cost_brl",
+      ]),
+    );
+    expect(columns("zaip_vw_daily_aggregates")).not.toContain(["sub", "run_count"].join(""));
+    expect(columns("zaip_vw_invocations")).toEqual(
+      expect.arrayContaining([
+        "cost_usd",
+        "cost_brl",
+        "cost_method",
+        "pricing_model",
+        "pricing_source",
+        "fx_rate",
+      ]),
+    );
+    expect(columns("zaip_vw_invocations")).not.toEqual(
+      expect.arrayContaining([
+        "coverage",
+        "_coverage",
+        "usage_segments_json",
+        "estimated_usd",
+        "estimated_brl",
+      ]),
+    );
+    expect(columns("zaip_vw_invocation_usage")).not.toContain("coverage");
+    expect(database.prepare("SELECT COUNT(*) AS count FROM zaip_vw_invocations").get()).toEqual({
+      count: 0,
+    });
+    database.close();
+    filesystem.close();
+  });
+});

package/templates/.agents/dashboard/packages/local-runtime/src/sqlite-read-model-schema.ts

@@ -0,0 +1,193 @@
+import { existsSync } from "node:fs";
+import { resolve } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { hasRequiredFirstFoldPayloads } from "./sqlite-first-fold-persistence.js";
+import type { SqliteReadModelSourceCheckpoint } from "./sqlite-read-model-source-fingerprint.js";
+export const dbRelativePath = ".local/sqlite/zaip-aios-dashboard.sqlite";
+// Schema 78 aligns usage-segment providers with canonical activity providers.
+// Old snapshots must rebuild even when their canonical hashes still match.
+export const readModelSchemaVersion = "78";
+export const requiredPersistedTables = [
+  "zaip_aios_activities",
+  "zaip_dim_llm_model",
+  "zaip_dim_provider",
+  "zaip_dim_zaip_agent",
+  "cost_invocation_dimension_facts",
+  "cost_group_daily_projection",
+  "cost_group_entity_projection",
+  "developer_cost_projection",
+  "operations_work_projection",
+] as const;
+export const requiredIndexes = [
+  "zaip_activities_agent_run",
+  "zaip_activities_relationships",
+  "zaip_activities_task_day",
+  "zaip_activities_spec_day",
+  "zaip_activities_prd_day",
+  "zaip_activities_agent",
+  "zaip_activities_run_id",
+  "zaip_activities_run_hierarchy",
+  "zaip_activities_period",
+  "zaip_activities_period_invocation",
+  "zaip_activities_status",
+  "zaip_activities_payload",
+  "zaip_activities_file_path",
+  "zaip_activities_work_item_identity",
+  "cost_dimension_facts_day_model",
+  "cost_dimension_facts_day_provider",
+  "cost_dimension_facts_invocation",
+  "cost_group_daily_projection_lookup",
+  "cost_group_entity_projection_lookup",
+  "developer_cost_projection_lookup",
+] as const;
+export const requiredViews = [
+  "zaip_vw_read_model_metadata",
+  "zaip_vw_operations_overview",
+  "zaip_vw_cost_overview",
+  "zaip_vw_cost_models",
+  "zaip_vw_cost_providers",
+  "zaip_vw_operations_agent_invocations",
+] as const;
+export const MAX_RECONCILED_RUN_INTERVAL_MS = 2 * 3_600_000;
+export function sqliteReadModelPath(workspaceRoot: string): string {
+  return resolve(process.env.ZAIP_SQLITE_DB_PATH ?? resolve(workspaceRoot, dbRelativePath));
+}
+
+export function persistedSourceCheckpoint(
+  path: string,
+): Map<string, SqliteReadModelSourceCheckpoint> {
+  // The first materialization has no persistent database to checkpoint.
+  // Opening SQLite read-only before this guard throws ERR_SQLITE_ERROR.
+  if (!existsSync(path)) return new Map();
+  let database: DatabaseSync | undefined;
+  try {
+    database = new DatabaseSync(path, { readOnly: true });
+    const rows = database
+      .prepare(
+        "SELECT source_path, source_hash, mtime_ms, ctime_ms, size_bytes, file_identity FROM zaip_vw_read_model_source_state",
+      )
+      .all() as Array<Record<string, unknown>>;
+    return new Map(
+      rows.flatMap((row) => {
+        const source = typeof row.source_path === "string" ? row.source_path : null;
+        const contentHash = typeof row.source_hash === "string" ? row.source_hash : null;
+        const fileIdentity = typeof row.file_identity === "string" ? row.file_identity : null;
+        const mtimeMs =
+          typeof row.mtime_ms === "number"
+            ? row.mtime_ms
+            : typeof row.mtime_ms === "string"
+              ? Number(row.mtime_ms)
+              : Number.NaN;
+        const ctimeMs =
+          typeof row.ctime_ms === "number"
+            ? row.ctime_ms
+            : typeof row.ctime_ms === "string"
+              ? Number(row.ctime_ms)
+              : Number.NaN;
+        if (
+          source === null ||
+          contentHash === null ||
+          fileIdentity === null ||
+          !Number.isFinite(mtimeMs) ||
+          !Number.isFinite(ctimeMs) ||
+          typeof row.size_bytes !== "number"
+        )
+          return [];
+        return [
+          [
+            source,
+            {
+              source,
+              mtimeMs,
+              ctimeMs,
+              size: row.size_bytes,
+              contentHash,
+              fileIdentity,
+            } satisfies SqliteReadModelSourceCheckpoint,
+          ],
+        ];
+      }),
+    );
+  } catch {
+    return new Map();
+  } finally {
+    database?.close();
+  }
+}
+
+/**
+ * Refuses stale or incomplete SQLite generations before they are served. This
+ * is intentionally independent of source fingerprints: a matching cache key
+ * is not evidence that a partially copied or old-schema file is safe.
+ */
+export function validateSqliteReadModelSchema(path: string): {
+  schemaVersion: string;
+  tableCount: number;
+  viewCount: number;
+  indexCount: number;
+} {
+  const database = new DatabaseSync(path, { readOnly: true });
+  try {
+    const integrity = database.prepare("PRAGMA integrity_check").get() as {
+      integrity_check?: string;
+    };
+    if (integrity.integrity_check !== "ok")
+      throw new Error(
+        `SQLite integrity check failed for ${path}: ${integrity.integrity_check ?? "unknown"}`,
+      );
+    const objects = database
+      .prepare("SELECT name, type FROM sqlite_master WHERE type IN ('table', 'view', 'index')")
+      .all() as Array<{ name: string; type: "table" | "view" | "index" }>;
+    const tables = objects
+      .filter((object) => object.type === "table")
+      .map((object) => object.name)
+      .sort();
+    const expectedTables = [...requiredPersistedTables].sort();
+    if (
+      tables.length !== expectedTables.length ||
+      tables.some((table, index) => table !== expectedTables[index])
+    )
+      throw new Error(
+        `SQLite schema ${readModelSchemaVersion} requires the complete persisted table contract; found ${tables.join(", ") || "none"}`,
+      );
+    const names = new Set(objects.map((object) => object.name));
+    const missingViews = requiredViews.filter((view) => !names.has(view));
+    if (missingViews.length > 0)
+      throw new Error(`SQLite schema is missing required views: ${missingViews.join(", ")}`);
+    const views = objects.filter((object) => object.type === "view").map((object) => object.name);
+    if (views.some((view) => !view.startsWith("zaip_vw_")))
+      throw new Error("SQLite persisted views must use the zaip_vw_ namespace");
+    const missingIndexes = requiredIndexes.filter((index) => !names.has(index));
+    if (missingIndexes.length > 0)
+      throw new Error(`SQLite schema is missing required indexes: ${missingIndexes.join(", ")}`);
+    const schemaVersion = (
+      database
+        .prepare("SELECT value FROM zaip_vw_read_model_metadata WHERE key = ?")
+        .get("schema_version") as { value?: string } | undefined
+    )?.value;
+    if (schemaVersion !== readModelSchemaVersion)
+      throw new Error(
+        `SQLite schema version must be ${readModelSchemaVersion}; found ${schemaVersion ?? "missing"}`,
+      );
+    return {
+      schemaVersion,
+      tableCount: tables.length,
+      viewCount: views.length,
+      indexCount: objects.filter(
+        (object) => object.type === "index" && !object.name.startsWith("sqlite_"),
+      ).length,
+    };
+  } finally {
+    database.close();
+  }
+}
+
+export function hasReusableSnapshotPayloads(database: DatabaseSync): boolean {
+  const indexes = database
+    .prepare("SELECT name FROM sqlite_master WHERE type='index'")
+    .all() as Array<{
+    name: string;
+  }>;
+  const names = new Set(indexes.map((index) => index.name));
+  return requiredIndexes.every((name) => names.has(name)) && hasRequiredFirstFoldPayloads(database);
+}

package/templates/.agents/dashboard/packages/local-runtime/src/sqlite-read-model-source-fingerprint.test.ts

@@ -1,6 +1,6 @@
 import { mkdir, mkdtemp, rename, rm, stat, utimes, writeFile } from "node:fs/promises";
 import { join, resolve } from "node:path";
-import { afterEach, describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
 import {
   type SqliteReadModelSourceCandidate,
   sourceFingerprint,
@@ -21,6 +21,7 @@ const checkpoints = (candidates: SqliteReadModelSourceCandidate[]) =>
   new Map(candidates.map((candidate) => [candidate.source, candidate]));
 
 afterEach(async () => {
+  vi.restoreAllMocks();
   await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
 });
 
@@ -49,6 +50,7 @@ describe("persistent source fingerprint checkpoints", () => {
 
   it("reuses historical content hashes across serialized checkpoints and detects additions and removals", async () => {
     const { root, path } = await fixture();
+    vi.spyOn(Date, "now").mockReturnValue(Date.now() + 5_000);
     const initial = await sourceFingerprint(root);
     const previous = checkpoints(JSON.parse(JSON.stringify(initial.candidates)));
     expect(initial.hashedFiles).toBe(1);
@@ -86,6 +88,57 @@ describe("persistent source fingerprint checkpoints", () => {
     expect(changed).toMatchObject({ hashedFiles: 1, reusedFiles: 0 });
   });
 
+  it("rehashes recent colliding metadata and the first settled checkpoint before reuse", async () => {
+    const { root, path } = await fixture();
+    const metadata = await stat(path);
+    const clock = vi.spyOn(Date, "now").mockReturnValue(metadata.ctimeMs);
+    const initial = await sourceFingerprint(root);
+    // Even a corrupted/stale hash with otherwise identical metadata cannot be
+    // trusted while writes can collide within the filesystem timestamp tick.
+    const stale = checkpoints(
+      initial.candidates.map((item) => ({ ...item, contentHash: "stale" })),
+    );
+    const recent = await sourceFingerprint(root, { previous: stale });
+    expect(recent).toMatchObject({ hashedFiles: 1, reusedFiles: 0 });
+    expect(recent.candidates[0]?.contentHash).toBe(initial.candidates[0]?.contentHash);
+    const serialized = checkpoints(JSON.parse(JSON.stringify(recent.candidates)));
+    clock.mockReturnValue(metadata.ctimeMs + 2_001);
+    const settled = await sourceFingerprint(root, { previous: serialized });
+    expect(settled).toMatchObject({ hashedFiles: 1, reusedFiles: 0 });
+    const reused = await sourceFingerprint(root, { previous: checkpoints(settled.candidates) });
+    expect(reused).toMatchObject({
+      hashedFiles: 0,
+      reusedFiles: 1,
+      fingerprint: settled.fingerprint,
+    });
+    const legacy = checkpoints(
+      settled.candidates.map((item) => ({
+        ...item,
+        contentHash: "stale",
+        fileIdentity: item.fileIdentity.replace("settled-v2:", ""),
+      })),
+    );
+    const upgraded = await sourceFingerprint(root, { previous: legacy });
+    expect(upgraded).toMatchObject({
+      hashedFiles: 1,
+      reusedFiles: 0,
+      fingerprint: settled.fingerprint,
+    });
+  });
+
+  it("detects rapid real same-size rewrites with restored mtime across repeated scans", async () => {
+    const { root, path } = await fixture();
+    const timestamp = new Date("2026-01-01T00:00:00Z");
+    let previous = await sourceFingerprint(root);
+    for (let iteration = 0; iteration < 20; iteration += 1) {
+      await writeFile(path, iteration % 2 ? "# first\n" : "# other\n");
+      await utimes(path, timestamp, timestamp);
+      const next = await sourceFingerprint(root, { previous: checkpoints(previous.candidates) });
+      expect(next.candidates[0]?.contentHash).not.toBe(previous.candidates[0]?.contentHash);
+      previous = next;
+    }
+  });
+
   it("detects atomic file replacement and allows explicit complete content verification", async () => {
     const { root, path } = await fixture();
     const initial = await sourceFingerprint(root);

package/templates/.agents/dashboard/packages/local-runtime/src/sqlite-read-model-source-fingerprint.ts

@@ -83,8 +83,10 @@ export async function sourceFingerprint(
   let hashedFiles = 0;
   let reusedFiles = 0;
   // Persisted checkpoints avoid rereading unchanged history across workers.
-  // ctime and identity detect same-size edits with preserved mtime and atomic
-  // replacement. Explicit verification bypasses this metadata shortcut.
+  // Windows can report identical ctime even in nanoseconds for rapid writes.
+  // Never cache a recent timestamp. Mark those hashes as unstable so that the
+  // first scan after the settling window also rehashes before allowing reuse.
+  // Explicit verification still bypasses this metadata shortcut entirely.
   for (let offset = 0; offset < candidates.length; offset += 32) {
     candidatesWithMetadata.push(
       ...(
@@ -94,17 +96,19 @@ export async function sourceFingerprint(
             try {
               for (let attempt = 0; attempt < 3; attempt += 1) {
                 const metadata = await stat(candidate);
+                const metadataSettled = Date.now() - metadata.ctimeMs > 2_000;
                 const current = {
                   path: candidate,
                   source,
                   mtimeMs: metadata.mtimeMs,
                   ctimeMs: metadata.ctimeMs,
                   size: metadata.size,
-                  fileIdentity: `${metadata.dev}:${metadata.ino}:${metadata.birthtimeMs}`,
+                  fileIdentity: `settled-v2:${metadata.dev}:${metadata.ino}:${metadata.birthtimeMs}${metadataSettled ? "" : ":recent"}`,
                 };
                 const previous = options.previous?.get(source);
                 if (
                   !options.verifyContent &&
+                  metadataSettled &&
                   previous !== undefined &&
                   previous.mtimeMs === current.mtimeMs &&
                   previous.ctimeMs === current.ctimeMs &&