@etiquekit/etq 1.0.11 → 1.0.13

package/packages/control/src/memory/sharded-drain-import.ts

@@ -0,0 +1,212 @@
+import { createHash } from 'node:crypto';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname, posix, resolve } from 'node:path';
+import { stringify as stringifyYaml } from 'yaml';
+import { drainImportMemoryCandidates, type MemoryDrainImportReceipt } from './drain-import';
+import { assertSharedMemoryScopeId, assertSharedMemoryText } from './sanitize';
+
+export interface ShardedDrainImportShard {
+  shardId: string;
+  sourceProjectRoot: string;
+  sourceWorkspaceRef?: string;
+}
+
+export interface ShardedMemoryDrainImportReceipt {
+  schema: 'memory-sharded-drain-import-receipt.v0';
+  receipt_id: string;
+  created_at: string;
+  topology: {
+    drain_topology: 'sharded_workspace_drains';
+    shard_key: 'workspace_or_team';
+    shard_count: number;
+    candidate_import_surface: 'candidate_import';
+    promotion_surface: 'authority_promotion_gate';
+    single_global_drain_is_demo_only: true;
+  };
+  source: {
+    source_store_id: string;
+    target_workspace_ref: string;
+    target_store_id: string;
+  };
+  shards: Array<{
+    shard_id: string;
+    source_workspace_ref: string;
+    source_store_id: string;
+    target_store_id: string;
+    child_receipt_id: string;
+    child_receipt_ref?: string;
+    candidate_count: number;
+    imported_count: number;
+  }>;
+  totals: {
+    candidate_count: number;
+    imported_count: number;
+  };
+  authority_boundary: {
+    sharded_drain_can_authorize_work: false;
+    sharded_drain_can_promote_candidates: false;
+    sharded_drain_can_write_parent_ledger: false;
+    child_drains_can_write_parent_ledger: false;
+    candidate_import_is_canonical_truth: false;
+    imported_entries_are_candidates: true;
+    promotion_gate_required: true;
+  };
+}
+
+export interface ShardedDrainImportMemoryCandidatesOptions {
+  targetProjectRoot: string;
+  sourceStoreId: string;
+  targetStoreId: string;
+  shards: ShardedDrainImportShard[];
+  targetStoresRoot?: string;
+  sourceStoresRoot?: string;
+  targetWorkspaceRef?: string;
+  receiptDir?: string;
+  receiptPath?: string;
+  at?: string;
+}
+
+function isoNow(at?: string): string {
+  const value = at ?? new Date().toISOString();
+  if (!Number.isFinite(Date.parse(value))) throw new Error('--at must be an ISO-8601 timestamp');
+  return new Date(value).toISOString();
+}
+
+function stableHash(value: unknown): string {
+  return createHash('sha256').update(JSON.stringify(value)).digest('hex').slice(0, 16);
+}
+
+function assertStoreId(value: string, field: string): string {
+  assertSharedMemoryScopeId(value, field);
+  return value;
+}
+
+function assertShardId(value: string): string {
+  assertSharedMemoryScopeId(value, 'shard_id');
+  return value;
+}
+
+function assertOpaque(value: string, field: string): string {
+  assertSharedMemoryText(value, field);
+  if (value.startsWith('/') || value.startsWith('\\') || /^[A-Za-z]:[\\/]/.test(value)) {
+    throw new Error(`${field} must be opaque or repo-relative, not a local absolute path`);
+  }
+  return value;
+}
+
+function safeRepoDir(raw: string | undefined): string | undefined {
+  if (!raw) return undefined;
+  if (raw.startsWith('/') || raw.startsWith('\\') || /^[A-Za-z]:[\\/]/.test(raw)) {
+    throw new Error('--receipt-dir must be repo-relative');
+  }
+  if (raw.includes('\\')) throw new Error('--receipt-dir must use POSIX / separators');
+  const normalized = posix.normalize(raw);
+  if (normalized === '.' || normalized === '..' || normalized.startsWith('../')) {
+    throw new Error('--receipt-dir must stay inside the target project');
+  }
+  assertSharedMemoryText(normalized, 'receipt_dir');
+  return normalized.replace(/\/+$/g, '');
+}
+
+function childReceiptRef(receiptDir: string | undefined, shardId: string): string | undefined {
+  return receiptDir ? `${receiptDir}/${shardId}.memory-drain-import-receipt.yaml` : undefined;
+}
+
+function childReceiptPath(targetProjectRoot: string, receiptRef: string | undefined): string | undefined {
+  return receiptRef ? resolve(targetProjectRoot, receiptRef) : undefined;
+}
+
+function shardPathPrefix(shardId: string, sourceStoreId: string): string {
+  return `imports/${shardId}/${sourceStoreId}`;
+}
+
+export async function shardedDrainImportMemoryCandidates(
+  options: ShardedDrainImportMemoryCandidatesOptions,
+): Promise<ShardedMemoryDrainImportReceipt> {
+  const createdAt = isoNow(options.at);
+  const targetProjectRoot = resolve(options.targetProjectRoot);
+  const sourceStoreId = assertStoreId(options.sourceStoreId, 'source_store');
+  const targetStoreId = assertStoreId(options.targetStoreId, 'target_store');
+  const targetWorkspaceRef = assertOpaque(options.targetWorkspaceRef ?? 'org-workspace', 'target_workspace_ref');
+  const receiptDir = safeRepoDir(options.receiptDir);
+  if (options.shards.length === 0) throw new Error('--shard must include at least one id=path entry');
+
+  const seen = new Set<string>();
+  const childReceipts: Array<{ shardId: string; sourceWorkspaceRef: string; receipt: MemoryDrainImportReceipt; receiptRef?: string }> = [];
+  for (const shard of options.shards) {
+    const shardId = assertShardId(shard.shardId);
+    if (seen.has(shardId)) throw new Error(`duplicate shard id: ${shardId}`);
+    seen.add(shardId);
+    const sourceWorkspaceRef = assertOpaque(shard.sourceWorkspaceRef ?? shardId, 'source_workspace_ref');
+    const receiptRef = childReceiptRef(receiptDir, shardId);
+    const receipt = await drainImportMemoryCandidates({
+      sourceProjectRoot: shard.sourceProjectRoot,
+      targetProjectRoot,
+      sourceStoresRoot: options.sourceStoresRoot,
+      targetStoresRoot: options.targetStoresRoot,
+      sourceStoreId,
+      targetStoreId,
+      sourceWorkspaceRef,
+      targetWorkspaceRef,
+      pathPrefix: shardPathPrefix(shardId, sourceStoreId),
+      receiptPath: childReceiptPath(targetProjectRoot, receiptRef),
+      at: createdAt,
+    });
+    childReceipts.push({ shardId, sourceWorkspaceRef, receipt, receiptRef });
+  }
+
+  const receipt: ShardedMemoryDrainImportReceipt = {
+    schema: 'memory-sharded-drain-import-receipt.v0',
+    receipt_id: `memory-sharded-drain-import:${stableHash({
+      targetWorkspaceRef,
+      sourceStoreId,
+      targetStoreId,
+      shards: childReceipts.map((shard) => `${shard.shardId}:${shard.receipt.receipt_id}`),
+    })}`,
+    created_at: createdAt,
+    topology: {
+      drain_topology: 'sharded_workspace_drains',
+      shard_key: 'workspace_or_team',
+      shard_count: childReceipts.length,
+      candidate_import_surface: 'candidate_import',
+      promotion_surface: 'authority_promotion_gate',
+      single_global_drain_is_demo_only: true,
+    },
+    source: {
+      source_store_id: sourceStoreId,
+      target_workspace_ref: targetWorkspaceRef,
+      target_store_id: targetStoreId,
+    },
+    shards: childReceipts.map((shard) => ({
+      shard_id: shard.shardId,
+      source_workspace_ref: shard.sourceWorkspaceRef,
+      source_store_id: sourceStoreId,
+      target_store_id: targetStoreId,
+      child_receipt_id: shard.receipt.receipt_id,
+      ...(shard.receiptRef ? { child_receipt_ref: shard.receiptRef } : {}),
+      candidate_count: shard.receipt.source.candidate_count,
+      imported_count: shard.receipt.target.imported_count,
+    })),
+    totals: {
+      candidate_count: childReceipts.reduce((sum, shard) => sum + shard.receipt.source.candidate_count, 0),
+      imported_count: childReceipts.reduce((sum, shard) => sum + shard.receipt.target.imported_count, 0),
+    },
+    authority_boundary: {
+      sharded_drain_can_authorize_work: false,
+      sharded_drain_can_promote_candidates: false,
+      sharded_drain_can_write_parent_ledger: false,
+      child_drains_can_write_parent_ledger: false,
+      candidate_import_is_canonical_truth: false,
+      imported_entries_are_candidates: true,
+      promotion_gate_required: true,
+    },
+  };
+
+  if (options.receiptPath) {
+    const receiptPath = resolve(options.receiptPath);
+    await mkdir(dirname(receiptPath), { recursive: true });
+    await writeFile(receiptPath, stringifyYaml(receipt));
+  }
+
+  return receipt;
+}

package/packages/control/src/memory/status.ts

@@ -0,0 +1,211 @@
+import { existsSync } from 'node:fs';
+import { readdir, readFile } from 'node:fs/promises';
+import { extname, join, resolve } from 'node:path';
+import { parse as parseYaml } from 'yaml';
+import { APPROVED_SCAN_DIRS, collectApprovedMemoryFiles, defaultMemoryDbPath } from './indexer';
+import { openMemoryStore, type MemoryStoreStats } from './store';
+import { defaultMemoryStoresRoot, listMemoryStores, type MemoryStoreSummary } from './store-lifecycle';
+
+const TEXT_EXTENSIONS = new Set(['.md', '.yaml', '.yml', '.json', '.jsonl', '.ndjson', '.txt']);
+
+export interface MemoryCapabilityStatus {
+  schema: 'memory-capability-status.v0';
+  module_status: 'experimental';
+  parity_posture: {
+    managed_runtime_filesystem_memory: 'partial';
+    mem0_style_semantic_memory: 'reserved';
+    full_feature_parity_claimed: false;
+  };
+  evidence_sources: {
+    approved_scan_dirs: {
+      path: string;
+      present: boolean;
+      file_count: number;
+    }[];
+    approved_file_count: number;
+    active_import_adapters: ['local_importer', 'manual'];
+    reserved_import_adapters: ['provider_memory_importers'];
+  };
+  recall_index: {
+    db_initialized: boolean;
+    search_adapter: 'sqlite_fts';
+    vector_adapter: 'reserved';
+    stats: MemoryStoreStats;
+  };
+  store_lifecycle: {
+    stores_root_initialized: boolean;
+    store_count: number;
+    candidate_entries: number;
+    rollups: number;
+    stores: MemoryStoreSummary[];
+  };
+  remote_writeback: {
+    mode: 'candidate_only';
+    drain_import_receipts: number;
+    drain_can_write_parent_ledger: false;
+  };
+  consolidation: {
+    status: 'candidate_rollups_available';
+    rollups: number;
+    marination_is_same_primitive: false;
+  };
+  authority_boundary: {
+    memory_can_authorize_work: false;
+    memory_can_close_task: false;
+    memory_can_merge: false;
+    memory_can_mutate_ledger: false;
+    memory_is_execution_truth: false;
+  };
+  missing_for_full_parity: string[];
+}
+
+async function countFilesUnder(root: string, relDir: string): Promise<number> {
+  const absDir = join(root, relDir);
+  if (!existsSync(absDir)) return 0;
+  let count = 0;
+  async function walk(dir: string): Promise<void> {
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      const abs = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (entry.name === 'node_modules' || entry.name === '.git' || entry.name === '_worktrees') continue;
+        await walk(abs);
+        continue;
+      }
+      if (entry.isFile() && TEXT_EXTENSIONS.has(extname(entry.name))) count += 1;
+    }
+  }
+  await walk(absDir);
+  return count;
+}
+
+async function countDrainImportReceipts(projectRoot: string): Promise<number> {
+  const receiptsRoot = join(projectRoot, 'docs', 'work', 'receipts');
+  if (!existsSync(receiptsRoot)) return 0;
+  let count = 0;
+  async function walk(dir: string): Promise<void> {
+    const entries = await readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      const abs = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        await walk(abs);
+        continue;
+      }
+      if (!entry.isFile() || !TEXT_EXTENSIONS.has(extname(entry.name))) continue;
+      try {
+        const raw = await readFile(abs, 'utf-8');
+        const parsed = parseYaml(raw) as unknown;
+        if (
+          typeof parsed === 'object'
+          && parsed !== null
+          && !Array.isArray(parsed)
+          && (parsed as Record<string, unknown>).schema === 'memory-drain-import-receipt.v0'
+        ) {
+          count += 1;
+        }
+      } catch {
+        // Status is advisory; malformed receipts should not prevent the operator
+        // from seeing the rest of the memory capability picture.
+      }
+    }
+  }
+  await walk(receiptsRoot);
+  return count;
+}
+
+function summarizeStores(stores: MemoryStoreSummary[]): { candidate_entries: number; rollups: number } {
+  return stores.reduce((totals, summary) => ({
+    candidate_entries: totals.candidate_entries + summary.counts.candidate_entries,
+    rollups: totals.rollups + summary.counts.rollups,
+  }), { candidate_entries: 0, rollups: 0 });
+}
+
+export async function memoryCapabilityStatus(options: {
+  projectRoot: string;
+  dbPath?: string;
+  storesRoot?: string;
+}): Promise<MemoryCapabilityStatus> {
+  const projectRoot = resolve(options.projectRoot);
+  const dbPath = options.dbPath ?? defaultMemoryDbPath(projectRoot);
+  const storesRoot = resolve(options.storesRoot ?? defaultMemoryStoresRoot(projectRoot));
+  const approvedFiles = await collectApprovedMemoryFiles(projectRoot);
+  const approvedScanDirs = await Promise.all(APPROVED_SCAN_DIRS.map(async (dir) => ({
+    path: dir,
+    present: existsSync(join(projectRoot, dir)),
+    file_count: await countFilesUnder(projectRoot, dir),
+  })));
+  let stats: MemoryStoreStats;
+  if (existsSync(dbPath)) {
+    const store = openMemoryStore(dbPath);
+    try {
+      stats = store.stats();
+    } finally {
+      store.close();
+    }
+  } else {
+    stats = {
+      candidates: {
+        total: 0,
+        pending: 0,
+        promoted: 0,
+        rejected: 0,
+      },
+      recall_capsules: 0,
+    };
+  }
+  const stores = await listMemoryStores({ projectRoot, storesRoot });
+  const storeCounts = summarizeStores(stores);
+  const drainImportReceipts = await countDrainImportReceipts(projectRoot);
+
+  return {
+    schema: 'memory-capability-status.v0',
+    module_status: 'experimental',
+    parity_posture: {
+      managed_runtime_filesystem_memory: 'partial',
+      mem0_style_semantic_memory: 'reserved',
+      full_feature_parity_claimed: false,
+    },
+    evidence_sources: {
+      approved_scan_dirs: approvedScanDirs,
+      approved_file_count: approvedFiles.length,
+      active_import_adapters: ['local_importer', 'manual'],
+      reserved_import_adapters: ['provider_memory_importers'],
+    },
+    recall_index: {
+      db_initialized: existsSync(dbPath),
+      search_adapter: 'sqlite_fts',
+      vector_adapter: 'reserved',
+      stats,
+    },
+    store_lifecycle: {
+      stores_root_initialized: existsSync(storesRoot),
+      store_count: stores.length,
+      candidate_entries: storeCounts.candidate_entries,
+      rollups: storeCounts.rollups,
+      stores,
+    },
+    remote_writeback: {
+      mode: 'candidate_only',
+      drain_import_receipts: drainImportReceipts,
+      drain_can_write_parent_ledger: false,
+    },
+    consolidation: {
+      status: 'candidate_rollups_available',
+      rollups: storeCounts.rollups,
+      marination_is_same_primitive: false,
+    },
+    authority_boundary: {
+      memory_can_authorize_work: false,
+      memory_can_close_task: false,
+      memory_can_merge: false,
+      memory_can_mutate_ledger: false,
+      memory_is_execution_truth: false,
+    },
+    missing_for_full_parity: [
+      'semantic vector adapter',
+      'scheduled curator promotion job',
+      'org_remote memory promotion policy UI',
+      'provider memory importer adapters',
+    ],
+  };
+}