@zaip-ai/cli 2.2.0 → 2.6.0

package/templates/.agents/dashboard/packages/local-runtime/src/sqlite-read-model-rebuilds-disposable-snapshot-serves-zaip-records.test.ts

@@ -0,0 +1,276 @@
+import { mkdir, mkdtemp, readFile, utimes, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { rebuildSqliteReadModel, SqliteReadModelFileSystem } from "./sqlite-read-model.js";
+
+const roots: string[] = [];
+const sqliteTestTempRoot = join(
+  resolve(import.meta.dirname, "../../../../../.."),
+  ".tmp",
+  "sqlite-read-model-tests",
+);
+async function createSqliteTestRoot(): Promise<string> {
+  await mkdir(sqliteTestTempRoot, { recursive: true });
+  return mkdtemp(join(sqliteTestTempRoot, "fixture-"));
+}
+afterEach(async () => {
+  vi.useRealTimers();
+  vi.restoreAllMocks();
+  await Promise.all(
+    roots
+      .splice(0)
+      .map((root) =>
+        import("node:fs/promises").then(({ rm }) => rm(root, { recursive: true, force: true })),
+      ),
+  );
+});
+describe("SQLite dashboard read model", () => {
+  it("rebuilds a disposable snapshot and serves ZAIP records from SQLite", async () => {
+    const root = await createSqliteTestRoot();
+    roots.push(root);
+    await mkdir(join(root, ".agents", "records", "tasks"), { recursive: true });
+    await mkdir(join(root, ".agents", "config"), { recursive: true });
+    await writeFile(join(root, ".agents", "records", "tasks", "task.md"), "# task\n", "utf8");
+    await writeFile(
+      join(root, ".agents", "config", "project.yaml"),
+      "zaip:\n  enabled: true\n",
+      "utf8",
+    );
+    const result = await rebuildSqliteReadModel(root);
+    const filesystem = new SqliteReadModelFileSystem(root);
+    expect(result.files).toBe(2);
+    await expect(filesystem.readText(".agents/records/tasks/task.md")).resolves.toContain("task");
+    await expect(filesystem.listFiles(".agents/records", ".md")).resolves.toEqual([
+      ".agents/records/tasks/task.md",
+    ]);
+    filesystem.close();
+  });
+  it("serves canonical Task trace inputs without lifecycle manifests", async () => {
+    const root = await createSqliteTestRoot();
+    roots.push(root);
+    await mkdir(join(root, ".agents", "records", "tasks", "2026", "08"), {
+      recursive: true,
+    });
+    await mkdir(join(root, ".agents", "config"), { recursive: true });
+    await writeFile(
+      join(root, ".agents", "config", "project.yaml"),
+      "zaip:\n  enabled: true\n",
+      "utf8",
+    );
+    await writeFile(
+      join(root, ".agents", "records", "tasks", "2026", "08", "TASK-TRACE.md"),
+      "---\nid: TASK-TRACE\ntitle: Canonical trace\nstatus: completed\nrisk: standard\ncreated_at: 2026-08-01T12:00:00.000Z\n---\n# Canonical trace\n",
+      "utf8",
+    );
+    await rebuildSqliteReadModel(root);
+    const filesystem = new SqliteReadModelFileSystem(root);
+    expect(filesystem.canonicalTraceFiles(["TASK-TRACE"])).toMatchObject([
+      { path: ".agents/records/tasks/2026/08/TASK-TRACE.md" },
+    ]);
+    expect(filesystem.taskTraceUsage("TASK-TRACE")).toEqual({
+      run_count: 0,
+      duration_ms: { value: null, known_runs: 0, unknown_runs: 0, coverage_ratio: 1 },
+      total_tokens: { value: null, known_runs: 0, unknown_runs: 0, coverage_ratio: 1 },
+      cost: {
+        value: null,
+        known_runs: 0,
+        unknown_runs: 0,
+        coverage_ratio: 1,
+        currency: null,
+      },
+    });
+    filesystem.close();
+  });
+  it("invalidates a nested canonical source change and ignores legacy aggregate telemetry", async () => {
+    const root = await createSqliteTestRoot();
+    roots.push(root);
+    const nested = join(root, ".agents", "records", "tasks", "nested", "task.md");
+    await mkdir(join(root, ".agents", "config"), { recursive: true });
+    await mkdir(join(root, ".agents", "records", "tasks", "nested"), { recursive: true });
+    await writeFile(
+      join(root, ".agents", "config", "project.yaml"),
+      "zaip:\n  enabled: true\n",
+      "utf8",
+    );
+    await writeFile(nested, "# before\n", "utf8");
+    await rebuildSqliteReadModel(root);
+    await writeFile(nested, "# after\n", "utf8");
+    const changedAt = new Date(Date.now() + 2_000);
+    await utimes(nested, changedAt, changedAt);
+    await rebuildSqliteReadModel(root);
+    const updated = new SqliteReadModelFileSystem(root);
+    await expect(updated.readText(".agents/records/tasks/nested/task.md")).resolves.toContain(
+      "after",
+    );
+    updated.close();
+
+    const duplicate = join(
+      root,
+      ".agents",
+      "records",
+      "telemetry",
+      "2026",
+      "08",
+      "writers",
+      "w",
+      "chunks",
+      "c.jsonl",
+    );
+    await mkdir(
+      join(root, ".agents", "records", "telemetry", "2026", "08", "writers", "w", "chunks"),
+      { recursive: true },
+    );
+    await writeFile(
+      join(root, ".agents", "records", "telemetry", "2026", "08", "writers", "w", "manifest.json"),
+      JSON.stringify({
+        chunks: [{ path: ".agents/records/telemetry/2026/08/writers/w/chunks/c.jsonl" }],
+      }),
+      "utf8",
+    );
+    await writeFile(
+      duplicate,
+      `${JSON.stringify({ agent_invocation_id: "same", summary_hash: "one" })}\n${JSON.stringify({ agent_invocation_id: "same", summary_hash: "two" })}\n`,
+      "utf8",
+    );
+    await expect(rebuildSqliteReadModel(root)).resolves.toMatchObject({ files: 2 });
+    expect(await readFile(nested, "utf8")).toContain("after");
+    const retained = new SqliteReadModelFileSystem(root);
+    await expect(retained.readText(".agents/records/tasks/nested/task.md")).resolves.toContain(
+      "after",
+    );
+    retained.close();
+  });
+  it("does not promote legacy aggregate telemetry into canonical activity facts", async () => {
+    const root = await createSqliteTestRoot();
+    roots.push(root);
+    await mkdir(
+      join(root, ".agents", "records", "telemetry", "2026", "08", "writers", "w", "chunks"),
+      { recursive: true },
+    );
+    await mkdir(join(root, ".agents", "config"), { recursive: true });
+    await writeFile(
+      join(root, ".agents", "config", "project.yaml"),
+      "zaip:\n  enabled: true\n",
+      "utf8",
+    );
+    const summary = {
+      agent_invocation_id: "root",
+      summary_hash: "hash",
+      root_agent_invocation_id: "root",
+      parent_agent_invocation_id: null,
+      execution_id: null,
+      task_id: null,
+      spec_id: null,
+      prd_id: null,
+      change_id: null,
+      run_id: "RUN-1",
+      occurred_at: "2026-08-01T12:00:00.000Z",
+      invocation_kind: "orchestrator",
+      delegation_depth: 0,
+      source_event_count: 1,
+      historical_cost: {
+        estimated_usd: 2,
+        estimated_brl: 10,
+        fx_rate: 5,
+        pricing_model: "model",
+        pricing_source: "canonical",
+      },
+      quantitative_metrics: { duration_ms: { known_sum: 5 }, tokens_total: { known_sum: 0 } },
+      usage_segments: [
+        {
+          provider: "openai",
+          model: "model",
+          service_tier: "default",
+          reasoning_effort: "medium",
+          tokens: { input: 0, cached_input: 0, output: 0, reasoning: 0, total: 0 },
+          historical_cost: {
+            estimated_usd: 2,
+            estimated_brl: 10,
+            fx_rate: 5,
+            pricing_model: "model",
+            pricing_source: "canonical",
+          },
+        },
+      ],
+    };
+    await writeFile(
+      join(
+        root,
+        ".agents",
+        "records",
+        "telemetry",
+        "2026",
+        "08",
+        "writers",
+        "w",
+        "chunks",
+        "c.jsonl",
+      ),
+      `${JSON.stringify(summary)}\n`,
+      "utf8",
+    );
+    await writeFile(
+      join(root, ".agents", "records", "telemetry", "2026", "08", "writers", "w", "manifest.json"),
+      JSON.stringify({
+        chunks: [{ path: ".agents/records/telemetry/2026/08/writers/w/chunks/c.jsonl" }],
+      }),
+      "utf8",
+    );
+    await rebuildSqliteReadModel(root);
+    const filesystem = new SqliteReadModelFileSystem(root);
+    expect(filesystem.firstFoldAggregate({ from: "2026-08-01", to: "2026-08-01" })).toEqual({
+      days: 0,
+      total_run_count: 0,
+      agent_invocation_count: 0,
+      llm_call_count: null,
+      duration_ms: null,
+      total_tokens: null,
+      cost_brl: null,
+      execution_day_count: 0,
+    });
+    const database = new DatabaseSync(resolve(root, ".local/sqlite/zaip-aios-dashboard.sqlite"), {
+      readOnly: true,
+    });
+    const columns = (table: string) =>
+      (database.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map(
+        (row) => row.name,
+      );
+    expect(columns("zaip_vw_daily_aggregates")).toEqual(
+      expect.arrayContaining([
+        "day",
+        "invocation_count",
+        "total_duration",
+        "total_tokens",
+        "total_cost_usd",
+        "total_cost_brl",
+      ]),
+    );
+    expect(columns("zaip_vw_daily_aggregates")).not.toContain(["sub", "run_count"].join(""));
+    expect(columns("zaip_vw_invocations")).toEqual(
+      expect.arrayContaining([
+        "cost_usd",
+        "cost_brl",
+        "cost_method",
+        "pricing_model",
+        "pricing_source",
+        "fx_rate",
+      ]),
+    );
+    expect(columns("zaip_vw_invocations")).not.toEqual(
+      expect.arrayContaining([
+        "coverage",
+        "_coverage",
+        "usage_segments_json",
+        "estimated_usd",
+        "estimated_brl",
+      ]),
+    );
+    expect(columns("zaip_vw_invocation_usage")).not.toContain("coverage");
+    expect(database.prepare("SELECT COUNT(*) AS count FROM zaip_vw_invocations").get()).toEqual({
+      count: 0,
+    });
+    database.close();
+    filesystem.close();
+  });
+});

package/templates/.agents/dashboard/packages/local-runtime/src/sqlite-read-model-schema.ts

@@ -0,0 +1,193 @@
+import { existsSync } from "node:fs";
+import { resolve } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { hasRequiredFirstFoldPayloads } from "./sqlite-first-fold-persistence.js";
+import type { SqliteReadModelSourceCheckpoint } from "./sqlite-read-model-source-fingerprint.js";
+export const dbRelativePath = ".local/sqlite/zaip-aios-dashboard.sqlite";
+// Schema 75 adds the persisted latest positive-token collection timestamp.
+// A schema rebuild is required even when source hashes match so an older
+// snapshot cannot masquerade as a telemetry-fresh projection.
+export const readModelSchemaVersion = "75";
+export const requiredPersistedTables = [
+  "zaip_aios_activities",
+  "zaip_dim_llm_model",
+  "zaip_dim_provider",
+  "zaip_dim_zaip_agent",
+  "cost_invocation_dimension_facts",
+  "cost_group_daily_projection",
+  "cost_group_entity_projection",
+  "developer_cost_projection",
+  "operations_work_projection",
+] as const;
+export const requiredIndexes = [
+  "zaip_activities_agent_run",
+  "zaip_activities_relationships",
+  "zaip_activities_task_day",
+  "zaip_activities_spec_day",
+  "zaip_activities_prd_day",
+  "zaip_activities_agent",
+  "zaip_activities_run_id",
+  "zaip_activities_run_hierarchy",
+  "zaip_activities_period",
+  "zaip_activities_period_invocation",
+  "zaip_activities_status",
+  "zaip_activities_payload",
+  "zaip_activities_work_item_identity",
+  "cost_dimension_facts_day_model",
+  "cost_dimension_facts_day_provider",
+  "cost_dimension_facts_invocation",
+  "cost_group_daily_projection_lookup",
+  "cost_group_entity_projection_lookup",
+  "developer_cost_projection_lookup",
+] as const;
+export const requiredViews = [
+  "zaip_vw_read_model_metadata",
+  "zaip_vw_operations_overview",
+  "zaip_vw_cost_overview",
+  "zaip_vw_cost_models",
+  "zaip_vw_cost_providers",
+  "zaip_vw_operations_agent_invocations",
+] as const;
+export const MAX_RECONCILED_RUN_INTERVAL_MS = 2 * 3_600_000;
+export function sqliteReadModelPath(workspaceRoot: string): string {
+  return resolve(process.env.ZAIP_SQLITE_DB_PATH ?? resolve(workspaceRoot, dbRelativePath));
+}
+
+export function persistedSourceCheckpoint(
+  path: string,
+): Map<string, SqliteReadModelSourceCheckpoint> {
+  // The first materialization has no persistent database to checkpoint.
+  // Opening SQLite read-only before this guard throws ERR_SQLITE_ERROR.
+  if (!existsSync(path)) return new Map();
+  let database: DatabaseSync | undefined;
+  try {
+    database = new DatabaseSync(path, { readOnly: true });
+    const rows = database
+      .prepare(
+        "SELECT source_path, source_hash, mtime_ms, ctime_ms, size_bytes, file_identity FROM zaip_vw_read_model_source_state",
+      )
+      .all() as Array<Record<string, unknown>>;
+    return new Map(
+      rows.flatMap((row) => {
+        const source = typeof row.source_path === "string" ? row.source_path : null;
+        const contentHash = typeof row.source_hash === "string" ? row.source_hash : null;
+        const fileIdentity = typeof row.file_identity === "string" ? row.file_identity : null;
+        const mtimeMs =
+          typeof row.mtime_ms === "number"
+            ? row.mtime_ms
+            : typeof row.mtime_ms === "string"
+              ? Number(row.mtime_ms)
+              : Number.NaN;
+        const ctimeMs =
+          typeof row.ctime_ms === "number"
+            ? row.ctime_ms
+            : typeof row.ctime_ms === "string"
+              ? Number(row.ctime_ms)
+              : Number.NaN;
+        if (
+          source === null ||
+          contentHash === null ||
+          fileIdentity === null ||
+          !Number.isFinite(mtimeMs) ||
+          !Number.isFinite(ctimeMs) ||
+          typeof row.size_bytes !== "number"
+        )
+          return [];
+        return [
+          [
+            source,
+            {
+              source,
+              mtimeMs,
+              ctimeMs,
+              size: row.size_bytes,
+              contentHash,
+              fileIdentity,
+            } satisfies SqliteReadModelSourceCheckpoint,
+          ],
+        ];
+      }),
+    );
+  } catch {
+    return new Map();
+  } finally {
+    database?.close();
+  }
+}
+
+/**
+ * Refuses stale or incomplete SQLite generations before they are served. This
+ * is intentionally independent of source fingerprints: a matching cache key
+ * is not evidence that a partially copied or old-schema file is safe.
+ */
+export function validateSqliteReadModelSchema(path: string): {
+  schemaVersion: string;
+  tableCount: number;
+  viewCount: number;
+  indexCount: number;
+} {
+  const database = new DatabaseSync(path, { readOnly: true });
+  try {
+    const integrity = database.prepare("PRAGMA integrity_check").get() as {
+      integrity_check?: string;
+    };
+    if (integrity.integrity_check !== "ok")
+      throw new Error(
+        `SQLite integrity check failed for ${path}: ${integrity.integrity_check ?? "unknown"}`,
+      );
+    const objects = database
+      .prepare("SELECT name, type FROM sqlite_master WHERE type IN ('table', 'view', 'index')")
+      .all() as Array<{ name: string; type: "table" | "view" | "index" }>;
+    const tables = objects
+      .filter((object) => object.type === "table")
+      .map((object) => object.name)
+      .sort();
+    const expectedTables = [...requiredPersistedTables].sort();
+    if (
+      tables.length !== expectedTables.length ||
+      tables.some((table, index) => table !== expectedTables[index])
+    )
+      throw new Error(
+        `SQLite schema ${readModelSchemaVersion} requires the complete persisted table contract; found ${tables.join(", ") || "none"}`,
+      );
+    const names = new Set(objects.map((object) => object.name));
+    const missingViews = requiredViews.filter((view) => !names.has(view));
+    if (missingViews.length > 0)
+      throw new Error(`SQLite schema is missing required views: ${missingViews.join(", ")}`);
+    const views = objects.filter((object) => object.type === "view").map((object) => object.name);
+    if (views.some((view) => !view.startsWith("zaip_vw_")))
+      throw new Error("SQLite persisted views must use the zaip_vw_ namespace");
+    const missingIndexes = requiredIndexes.filter((index) => !names.has(index));
+    if (missingIndexes.length > 0)
+      throw new Error(`SQLite schema is missing required indexes: ${missingIndexes.join(", ")}`);
+    const schemaVersion = (
+      database
+        .prepare("SELECT value FROM zaip_vw_read_model_metadata WHERE key = ?")
+        .get("schema_version") as { value?: string } | undefined
+    )?.value;
+    if (schemaVersion !== readModelSchemaVersion)
+      throw new Error(
+        `SQLite schema version must be ${readModelSchemaVersion}; found ${schemaVersion ?? "missing"}`,
+      );
+    return {
+      schemaVersion,
+      tableCount: tables.length,
+      viewCount: views.length,
+      indexCount: objects.filter(
+        (object) => object.type === "index" && !object.name.startsWith("sqlite_"),
+      ).length,
+    };
+  } finally {
+    database.close();
+  }
+}
+
+export function hasReusableSnapshotPayloads(database: DatabaseSync): boolean {
+  const indexes = database
+    .prepare("SELECT name FROM sqlite_master WHERE type='index'")
+    .all() as Array<{
+    name: string;
+  }>;
+  const names = new Set(indexes.map((index) => index.name));
+  return requiredIndexes.every((name) => names.has(name)) && hasRequiredFirstFoldPayloads(database);
+}

package/templates/.agents/dashboard/packages/local-runtime/src/sqlite-read-model-types.ts

@@ -0,0 +1,123 @@
+export type SqliteFirstFoldAggregate = {
+  days: number;
+  total_run_count: number;
+  agent_invocation_count: number;
+  llm_call_count: number | null;
+  duration_ms: number | null;
+  total_tokens: number | null;
+  cost_brl: number | null;
+  execution_day_count: number;
+};
+export type {
+  SqliteFirstFoldAllTime,
+  SqliteFirstFoldAllTimeBoard,
+  SqliteFirstFoldMonthlyQuick,
+} from "./sqlite-read-model-first-fold-types.js";
+
+import type { SqliteFirstFoldAllTime } from "./sqlite-read-model-first-fold-types.js";
+
+/** The only all-time facts needed before the Operations first fold is painted. */
+export type SqliteFirstFoldViewport = Pick<
+  SqliteFirstFoldAllTime,
+  "aggregate" | "task_status_counts" | "task_risk_counts"
+>;
+
+export type SqliteSummaryKind = "task" | "spec" | "prd";
+
+export const sqliteViewMetadata = {
+  cost_monitor_daily: { date: "period_key", id: "period_key", order: "period_key DESC" },
+  cost_monitor_monthly: { date: "period_key", id: "period_key", order: "period_key DESC" },
+  cost_monitor_annual: { date: "period_key", id: "period_key", order: "period_key DESC" },
+  cost_monitor_overview_view: { date: "period_key", id: "period_key", order: "period_key DESC" },
+  cost_monitor_executions: {
+    date: null,
+    id: "execution_id",
+    order: "invocation_count DESC, execution_id DESC",
+  },
+  cost_monitor_agent_invocations: {
+    date: "occurred_day_sp",
+    id: "agent_invocation_id",
+    order: "occurred_at DESC, agent_invocation_id DESC",
+  },
+  cost_monitor_zaip_agents: {
+    date: null,
+    id: "zaip_agent",
+    order: "invocation_count DESC, zaip_agent",
+  },
+  cost_monitor_models: { date: null, id: "model", order: "invocation_count DESC, model" },
+  cost_monitor_providers: { date: null, id: "provider", order: "invocation_count DESC, provider" },
+  cost_monitor_tasks: { date: null, id: "task_id", order: "invocation_count DESC, task_id" },
+  cost_monitor_specs: { date: null, id: "spec_id", order: "invocation_count DESC, spec_id" },
+  cost_monitor_prds: { date: null, id: "prd_id", order: "invocation_count DESC, prd_id" },
+  cost_monitor_conversations: {
+    date: null,
+    id: "conversation_id",
+    order: "invocation_count DESC, conversation_id",
+  },
+  operations_work_list: { date: "occurred_day_sp", id: "id", order: "occurred_at DESC, id DESC" },
+  operations_work_kanban: { date: "occurred_day_sp", id: "id", order: "occurred_at DESC, id DESC" },
+  operations_executions: {
+    date: null,
+    id: "execution_id",
+    order: "started_at DESC, execution_id DESC",
+  },
+  operations_agent_invocations: {
+    date: "source_day_sp",
+    id: "agent_invocation_id",
+    order: "occurred_at DESC, agent_invocation_id DESC",
+  },
+  operations_conversations: {
+    date: null,
+    id: "conversation_id",
+    order: "started_at DESC, conversation_id DESC",
+  },
+  operations_decisions: { date: "occurred_day_sp", id: "id", order: "occurred_at DESC, id DESC" },
+  operations_changes: { date: "occurred_day_sp", id: "id", order: "occurred_at DESC, id DESC" },
+  operations_daily: { date: "day", id: "day", order: "day DESC" },
+  operations_monthly: { date: "period_key", id: "period_key", order: "period_key DESC" },
+  operations_annual: { date: "period_key", id: "period_key", order: "period_key DESC" },
+  first_fold_overview: { date: "start_day", id: "period_key", order: "period_key DESC" },
+  first_fold_task_status: { date: "day", id: "status", order: "day DESC, status" },
+  first_fold_task_risk: { date: "day", id: "risk", order: "day DESC, risk" },
+  first_fold_top_tasks: {
+    date: "occurred_day_sp",
+    id: "id",
+    order: "total_duration DESC, id DESC",
+  },
+  first_fold_overview_all_time: { date: null, id: "period_key", order: "period_key" },
+  first_fold_task_status_all_time: { date: null, id: "status", order: "status" },
+  first_fold_task_risk_all_time: { date: null, id: "risk", order: "risk" },
+  first_fold_task_metrics_all_time: { date: null, id: null, order: "task_count DESC" },
+  first_fold_top_tasks_all_time: { date: null, id: "task_id", order: "rank" },
+  first_fold_spec_status_all_time: { date: null, id: "stage", order: "stage" },
+  first_fold_spec_classification_all_time: {
+    date: null,
+    id: "classification",
+    order: "classification",
+  },
+  first_fold_spec_metrics_all_time: { date: null, id: null, order: "entity_count DESC" },
+  first_fold_top_specs_all_time: { date: null, id: "entity_id", order: "rank" },
+  first_fold_prd_status_all_time: { date: null, id: "stage", order: "stage" },
+  first_fold_prd_classification_all_time: {
+    date: null,
+    id: "classification",
+    order: "classification",
+  },
+  first_fold_prd_metrics_all_time: { date: null, id: null, order: "entity_count DESC" },
+  first_fold_top_prds_all_time: { date: null, id: "entity_id", order: "rank" },
+  first_fold_task: { date: "occurred_day_sp", id: "id", order: "occurred_at DESC, id DESC" },
+  first_fold_spec: { date: "occurred_day_sp", id: "id", order: "occurred_at DESC, id DESC" },
+  first_fold_prd: { date: "occurred_day_sp", id: "id", order: "occurred_at DESC, id DESC" },
+} as const;
+export type SqliteViewName = keyof typeof sqliteViewMetadata;
+
+export type SqliteSummaryPage = {
+  data_source: "sqlite_read_model";
+  kind: SqliteSummaryKind;
+  offset: number;
+  limit: number;
+  total: number;
+  has_more: boolean;
+  next_cursor: string | null;
+  rows: Array<Record<string, unknown>>;
+};