@etiquekit/etq 1.0.14 → 1.0.15

package/packages/control/src/memory/sharded-drain-import.ts

@@ -1,212 +0,0 @@
-import { createHash } from 'node:crypto';
-import { mkdir, writeFile } from 'node:fs/promises';
-import { dirname, posix, resolve } from 'node:path';
-import { stringify as stringifyYaml } from 'yaml';
-import { drainImportMemoryCandidates, type MemoryDrainImportReceipt } from './drain-import';
-import { assertSharedMemoryScopeId, assertSharedMemoryText } from './sanitize';
-
-export interface ShardedDrainImportShard {
-  shardId: string;
-  sourceProjectRoot: string;
-  sourceWorkspaceRef?: string;
-}
-
-export interface ShardedMemoryDrainImportReceipt {
-  schema: 'memory-sharded-drain-import-receipt.v0';
-  receipt_id: string;
-  created_at: string;
-  topology: {
-    drain_topology: 'sharded_workspace_drains';
-    shard_key: 'workspace_or_team';
-    shard_count: number;
-    candidate_import_surface: 'candidate_import';
-    promotion_surface: 'authority_promotion_gate';
-    single_global_drain_is_demo_only: true;
-  };
-  source: {
-    source_store_id: string;
-    target_workspace_ref: string;
-    target_store_id: string;
-  };
-  shards: Array<{
-    shard_id: string;
-    source_workspace_ref: string;
-    source_store_id: string;
-    target_store_id: string;
-    child_receipt_id: string;
-    child_receipt_ref?: string;
-    candidate_count: number;
-    imported_count: number;
-  }>;
-  totals: {
-    candidate_count: number;
-    imported_count: number;
-  };
-  authority_boundary: {
-    sharded_drain_can_authorize_work: false;
-    sharded_drain_can_promote_candidates: false;
-    sharded_drain_can_write_parent_ledger: false;
-    child_drains_can_write_parent_ledger: false;
-    candidate_import_is_canonical_truth: false;
-    imported_entries_are_candidates: true;
-    promotion_gate_required: true;
-  };
-}
-
-export interface ShardedDrainImportMemoryCandidatesOptions {
-  targetProjectRoot: string;
-  sourceStoreId: string;
-  targetStoreId: string;
-  shards: ShardedDrainImportShard[];
-  targetStoresRoot?: string;
-  sourceStoresRoot?: string;
-  targetWorkspaceRef?: string;
-  receiptDir?: string;
-  receiptPath?: string;
-  at?: string;
-}
-
-function isoNow(at?: string): string {
-  const value = at ?? new Date().toISOString();
-  if (!Number.isFinite(Date.parse(value))) throw new Error('--at must be an ISO-8601 timestamp');
-  return new Date(value).toISOString();
-}
-
-function stableHash(value: unknown): string {
-  return createHash('sha256').update(JSON.stringify(value)).digest('hex').slice(0, 16);
-}
-
-function assertStoreId(value: string, field: string): string {
-  assertSharedMemoryScopeId(value, field);
-  return value;
-}
-
-function assertShardId(value: string): string {
-  assertSharedMemoryScopeId(value, 'shard_id');
-  return value;
-}
-
-function assertOpaque(value: string, field: string): string {
-  assertSharedMemoryText(value, field);
-  if (value.startsWith('/') || value.startsWith('\\') || /^[A-Za-z]:[\\/]/.test(value)) {
-    throw new Error(`${field} must be opaque or repo-relative, not a local absolute path`);
-  }
-  return value;
-}
-
-function safeRepoDir(raw: string | undefined): string | undefined {
-  if (!raw) return undefined;
-  if (raw.startsWith('/') || raw.startsWith('\\') || /^[A-Za-z]:[\\/]/.test(raw)) {
-    throw new Error('--receipt-dir must be repo-relative');
-  }
-  if (raw.includes('\\')) throw new Error('--receipt-dir must use POSIX / separators');
-  const normalized = posix.normalize(raw);
-  if (normalized === '.' || normalized === '..' || normalized.startsWith('../')) {
-    throw new Error('--receipt-dir must stay inside the target project');
-  }
-  assertSharedMemoryText(normalized, 'receipt_dir');
-  return normalized.replace(/\/+$/g, '');
-}
-
-function childReceiptRef(receiptDir: string | undefined, shardId: string): string | undefined {
-  return receiptDir ? `${receiptDir}/${shardId}.memory-drain-import-receipt.yaml` : undefined;
-}
-
-function childReceiptPath(targetProjectRoot: string, receiptRef: string | undefined): string | undefined {
-  return receiptRef ? resolve(targetProjectRoot, receiptRef) : undefined;
-}
-
-function shardPathPrefix(shardId: string, sourceStoreId: string): string {
-  return `imports/${shardId}/${sourceStoreId}`;
-}
-
-export async function shardedDrainImportMemoryCandidates(
-  options: ShardedDrainImportMemoryCandidatesOptions,
-): Promise<ShardedMemoryDrainImportReceipt> {
-  const createdAt = isoNow(options.at);
-  const targetProjectRoot = resolve(options.targetProjectRoot);
-  const sourceStoreId = assertStoreId(options.sourceStoreId, 'source_store');
-  const targetStoreId = assertStoreId(options.targetStoreId, 'target_store');
-  const targetWorkspaceRef = assertOpaque(options.targetWorkspaceRef ?? 'org-workspace', 'target_workspace_ref');
-  const receiptDir = safeRepoDir(options.receiptDir);
-  if (options.shards.length === 0) throw new Error('--shard must include at least one id=path entry');
-
-  const seen = new Set<string>();
-  const childReceipts: Array<{ shardId: string; sourceWorkspaceRef: string; receipt: MemoryDrainImportReceipt; receiptRef?: string }> = [];
-  for (const shard of options.shards) {
-    const shardId = assertShardId(shard.shardId);
-    if (seen.has(shardId)) throw new Error(`duplicate shard id: ${shardId}`);
-    seen.add(shardId);
-    const sourceWorkspaceRef = assertOpaque(shard.sourceWorkspaceRef ?? shardId, 'source_workspace_ref');
-    const receiptRef = childReceiptRef(receiptDir, shardId);
-    const receipt = await drainImportMemoryCandidates({
-      sourceProjectRoot: shard.sourceProjectRoot,
-      targetProjectRoot,
-      sourceStoresRoot: options.sourceStoresRoot,
-      targetStoresRoot: options.targetStoresRoot,
-      sourceStoreId,
-      targetStoreId,
-      sourceWorkspaceRef,
-      targetWorkspaceRef,
-      pathPrefix: shardPathPrefix(shardId, sourceStoreId),
-      receiptPath: childReceiptPath(targetProjectRoot, receiptRef),
-      at: createdAt,
-    });
-    childReceipts.push({ shardId, sourceWorkspaceRef, receipt, receiptRef });
-  }
-
-  const receipt: ShardedMemoryDrainImportReceipt = {
-    schema: 'memory-sharded-drain-import-receipt.v0',
-    receipt_id: `memory-sharded-drain-import:${stableHash({
-      targetWorkspaceRef,
-      sourceStoreId,
-      targetStoreId,
-      shards: childReceipts.map((shard) => `${shard.shardId}:${shard.receipt.receipt_id}`),
-    })}`,
-    created_at: createdAt,
-    topology: {
-      drain_topology: 'sharded_workspace_drains',
-      shard_key: 'workspace_or_team',
-      shard_count: childReceipts.length,
-      candidate_import_surface: 'candidate_import',
-      promotion_surface: 'authority_promotion_gate',
-      single_global_drain_is_demo_only: true,
-    },
-    source: {
-      source_store_id: sourceStoreId,
-      target_workspace_ref: targetWorkspaceRef,
-      target_store_id: targetStoreId,
-    },
-    shards: childReceipts.map((shard) => ({
-      shard_id: shard.shardId,
-      source_workspace_ref: shard.sourceWorkspaceRef,
-      source_store_id: sourceStoreId,
-      target_store_id: targetStoreId,
-      child_receipt_id: shard.receipt.receipt_id,
-      ...(shard.receiptRef ? { child_receipt_ref: shard.receiptRef } : {}),
-      candidate_count: shard.receipt.source.candidate_count,
-      imported_count: shard.receipt.target.imported_count,
-    })),
-    totals: {
-      candidate_count: childReceipts.reduce((sum, shard) => sum + shard.receipt.source.candidate_count, 0),
-      imported_count: childReceipts.reduce((sum, shard) => sum + shard.receipt.target.imported_count, 0),
-    },
-    authority_boundary: {
-      sharded_drain_can_authorize_work: false,
-      sharded_drain_can_promote_candidates: false,
-      sharded_drain_can_write_parent_ledger: false,
-      child_drains_can_write_parent_ledger: false,
-      candidate_import_is_canonical_truth: false,
-      imported_entries_are_candidates: true,
-      promotion_gate_required: true,
-    },
-  };
-
-  if (options.receiptPath) {
-    const receiptPath = resolve(options.receiptPath);
-    await mkdir(dirname(receiptPath), { recursive: true });
-    await writeFile(receiptPath, stringifyYaml(receipt));
-  }
-
-  return receipt;
-}

package/packages/control/src/memory/status.ts

@@ -1,211 +0,0 @@
-import { existsSync } from 'node:fs';
-import { readdir, readFile } from 'node:fs/promises';
-import { extname, join, resolve } from 'node:path';
-import { parse as parseYaml } from 'yaml';
-import { APPROVED_SCAN_DIRS, collectApprovedMemoryFiles, defaultMemoryDbPath } from './indexer';
-import { openMemoryStore, type MemoryStoreStats } from './store';
-import { defaultMemoryStoresRoot, listMemoryStores, type MemoryStoreSummary } from './store-lifecycle';
-
-const TEXT_EXTENSIONS = new Set(['.md', '.yaml', '.yml', '.json', '.jsonl', '.ndjson', '.txt']);
-
-export interface MemoryCapabilityStatus {
-  schema: 'memory-capability-status.v0';
-  module_status: 'experimental';
-  parity_posture: {
-    managed_runtime_filesystem_memory: 'partial';
-    mem0_style_semantic_memory: 'reserved';
-    full_feature_parity_claimed: false;
-  };
-  evidence_sources: {
-    approved_scan_dirs: {
-      path: string;
-      present: boolean;
-      file_count: number;
-    }[];
-    approved_file_count: number;
-    active_import_adapters: ['local_importer', 'manual'];
-    reserved_import_adapters: ['provider_memory_importers'];
-  };
-  recall_index: {
-    db_initialized: boolean;
-    search_adapter: 'sqlite_fts';
-    vector_adapter: 'reserved';
-    stats: MemoryStoreStats;
-  };
-  store_lifecycle: {
-    stores_root_initialized: boolean;
-    store_count: number;
-    candidate_entries: number;
-    rollups: number;
-    stores: MemoryStoreSummary[];
-  };
-  remote_writeback: {
-    mode: 'candidate_only';
-    drain_import_receipts: number;
-    drain_can_write_parent_ledger: false;
-  };
-  consolidation: {
-    status: 'candidate_rollups_available';
-    rollups: number;
-    marination_is_same_primitive: false;
-  };
-  authority_boundary: {
-    memory_can_authorize_work: false;
-    memory_can_close_task: false;
-    memory_can_merge: false;
-    memory_can_mutate_ledger: false;
-    memory_is_execution_truth: false;
-  };
-  missing_for_full_parity: string[];
-}
-
-async function countFilesUnder(root: string, relDir: string): Promise<number> {
-  const absDir = join(root, relDir);
-  if (!existsSync(absDir)) return 0;
-  let count = 0;
-  async function walk(dir: string): Promise<void> {
-    const entries = await readdir(dir, { withFileTypes: true });
-    for (const entry of entries) {
-      const abs = join(dir, entry.name);
-      if (entry.isDirectory()) {
-        if (entry.name === 'node_modules' || entry.name === '.git' || entry.name === '_worktrees') continue;
-        await walk(abs);
-        continue;
-      }
-      if (entry.isFile() && TEXT_EXTENSIONS.has(extname(entry.name))) count += 1;
-    }
-  }
-  await walk(absDir);
-  return count;
-}
-
-async function countDrainImportReceipts(projectRoot: string): Promise<number> {
-  const receiptsRoot = join(projectRoot, 'docs', 'work', 'receipts');
-  if (!existsSync(receiptsRoot)) return 0;
-  let count = 0;
-  async function walk(dir: string): Promise<void> {
-    const entries = await readdir(dir, { withFileTypes: true });
-    for (const entry of entries) {
-      const abs = join(dir, entry.name);
-      if (entry.isDirectory()) {
-        await walk(abs);
-        continue;
-      }
-      if (!entry.isFile() || !TEXT_EXTENSIONS.has(extname(entry.name))) continue;
-      try {
-        const raw = await readFile(abs, 'utf-8');
-        const parsed = parseYaml(raw) as unknown;
-        if (
-          typeof parsed === 'object'
-          && parsed !== null
-          && !Array.isArray(parsed)
-          && (parsed as Record<string, unknown>).schema === 'memory-drain-import-receipt.v0'
-        ) {
-          count += 1;
-        }
-      } catch {
-        // Status is advisory; malformed receipts should not prevent the operator
-        // from seeing the rest of the memory capability picture.
-      }
-    }
-  }
-  await walk(receiptsRoot);
-  return count;
-}
-
-function summarizeStores(stores: MemoryStoreSummary[]): { candidate_entries: number; rollups: number } {
-  return stores.reduce((totals, summary) => ({
-    candidate_entries: totals.candidate_entries + summary.counts.candidate_entries,
-    rollups: totals.rollups + summary.counts.rollups,
-  }), { candidate_entries: 0, rollups: 0 });
-}
-
-export async function memoryCapabilityStatus(options: {
-  projectRoot: string;
-  dbPath?: string;
-  storesRoot?: string;
-}): Promise<MemoryCapabilityStatus> {
-  const projectRoot = resolve(options.projectRoot);
-  const dbPath = options.dbPath ?? defaultMemoryDbPath(projectRoot);
-  const storesRoot = resolve(options.storesRoot ?? defaultMemoryStoresRoot(projectRoot));
-  const approvedFiles = await collectApprovedMemoryFiles(projectRoot);
-  const approvedScanDirs = await Promise.all(APPROVED_SCAN_DIRS.map(async (dir) => ({
-    path: dir,
-    present: existsSync(join(projectRoot, dir)),
-    file_count: await countFilesUnder(projectRoot, dir),
-  })));
-  let stats: MemoryStoreStats;
-  if (existsSync(dbPath)) {
-    const store = openMemoryStore(dbPath);
-    try {
-      stats = store.stats();
-    } finally {
-      store.close();
-    }
-  } else {
-    stats = {
-      candidates: {
-        total: 0,
-        pending: 0,
-        promoted: 0,
-        rejected: 0,
-      },
-      recall_capsules: 0,
-    };
-  }
-  const stores = await listMemoryStores({ projectRoot, storesRoot });
-  const storeCounts = summarizeStores(stores);
-  const drainImportReceipts = await countDrainImportReceipts(projectRoot);
-
-  return {
-    schema: 'memory-capability-status.v0',
-    module_status: 'experimental',
-    parity_posture: {
-      managed_runtime_filesystem_memory: 'partial',
-      mem0_style_semantic_memory: 'reserved',
-      full_feature_parity_claimed: false,
-    },
-    evidence_sources: {
-      approved_scan_dirs: approvedScanDirs,
-      approved_file_count: approvedFiles.length,
-      active_import_adapters: ['local_importer', 'manual'],
-      reserved_import_adapters: ['provider_memory_importers'],
-    },
-    recall_index: {
-      db_initialized: existsSync(dbPath),
-      search_adapter: 'sqlite_fts',
-      vector_adapter: 'reserved',
-      stats,
-    },
-    store_lifecycle: {
-      stores_root_initialized: existsSync(storesRoot),
-      store_count: stores.length,
-      candidate_entries: storeCounts.candidate_entries,
-      rollups: storeCounts.rollups,
-      stores,
-    },
-    remote_writeback: {
-      mode: 'candidate_only',
-      drain_import_receipts: drainImportReceipts,
-      drain_can_write_parent_ledger: false,
-    },
-    consolidation: {
-      status: 'candidate_rollups_available',
-      rollups: storeCounts.rollups,
-      marination_is_same_primitive: false,
-    },
-    authority_boundary: {
-      memory_can_authorize_work: false,
-      memory_can_close_task: false,
-      memory_can_merge: false,
-      memory_can_mutate_ledger: false,
-      memory_is_execution_truth: false,
-    },
-    missing_for_full_parity: [
-      'semantic vector adapter',
-      'scheduled curator promotion job',
-      'org_remote memory promotion policy UI',
-      'provider memory importer adapters',
-    ],
-  };
-}