@geraldmaron/construct 1.0.24 → 1.1.1

package/lib/ingest/chunker.mjs

@@ -1,94 +0,0 @@
-/**
- * lib/ingest/chunker.mjs — paragraph-boundary chunker with sentence overlap.
- *
- * 2026-06 best practice notes:
- *   - Recursive/sentence chunking remains the high-recall default (~85-90%
- *     on 2026 RAG benchmarks). Semantic chunking gains only ~2-3% recall at
- *     ~14× the embedding cost and only matters above ~5k token docs.
- *   - Parent-document retrieval is the meaningful 2026 upgrade beyond
- *     paragraph chunking — track separately if recall plateaus.
- *
- * Strategy:
- *   1. Split markdown by blank-line paragraph boundaries.
- *   2. Pack paragraphs into chunks under maxChars (default 1500 ≈ 375 tokens).
- *   3. Add 1-2 sentence overlap between consecutive chunks to preserve
- *      cross-chunk anchors.
- *   4. Preserve markdown structure: never split inside a fenced code block.
- */
-
-const DEFAULT_MAX_CHARS = 1500;
-const DEFAULT_OVERLAP_SENTENCES = 2;
-const SENTENCE_RE = /[^.!?]+[.!?]+["')\]]*\s*/g;
-
-export function splitSentences(text) {
-  const matches = text.match(SENTENCE_RE);
-  if (matches && matches.length) return matches.map((s) => s.trim()).filter(Boolean);
-  return text.split('\n').map((s) => s.trim()).filter(Boolean);
-}
-
-function tailSentences(text, count) {
-  const sentences = splitSentences(text);
-  return sentences.slice(-count).join(' ');
-}
-
-function splitParagraphs(markdown) {
-  const blocks = [];
-  let buffer = '';
-  let inFence = false;
-  for (const line of markdown.split('\n')) {
-    if (/^```/.test(line.trim())) inFence = !inFence;
-    if (!inFence && line.trim() === '' && buffer.trim()) {
-      blocks.push(buffer.trim());
-      buffer = '';
-      continue;
-    }
-    buffer += line + '\n';
-  }
-  if (buffer.trim()) blocks.push(buffer.trim());
-  return blocks;
-}
-
-export function chunkMarkdown(markdown, { maxChars = DEFAULT_MAX_CHARS, overlapSentences = DEFAULT_OVERLAP_SENTENCES } = {}) {
-  const paragraphs = splitParagraphs(markdown || '');
-  if (paragraphs.length === 0) return [];
-
-  const chunks = [];
-  let current = '';
-  let chunkIndex = 0;
-  const flushChunk = () => {
-    if (!current.trim()) return;
-    const text = current.trim();
-    chunks.push({
-      index: chunkIndex++,
-      text,
-      chars: text.length,
-    });
-    const overlap = overlapSentences > 0 ? tailSentences(text, overlapSentences) : '';
-    current = overlap ? overlap + '\n\n' : '';
-  };
-
-  for (const para of paragraphs) {
-    if (current.length + para.length + 2 > maxChars && current.trim()) {
-      flushChunk();
-    }
-    if (para.length > maxChars) {
-      flushChunk();
-      const sentences = splitSentences(para);
-      let buf = '';
-      for (const s of sentences) {
-        if (buf.length + s.length + 1 > maxChars && buf.trim()) {
-          chunks.push({ index: chunkIndex++, text: buf.trim(), chars: buf.trim().length });
-          buf = (overlapSentences > 0 ? tailSentences(buf, overlapSentences) : '') + (overlapSentences > 0 ? ' ' : '');
-        }
-        buf += s + ' ';
-      }
-      if (buf.trim()) {
-        current = buf.trim() + '\n\n';
-      }
-      continue;
-    }
-    current += (current ? '\n\n' : '') + para;
-  }
-  flushChunk();
-  return chunks;
-}

package/lib/ingest/pipeline.mjs

@@ -1,53 +0,0 @@
-/**
- * lib/ingest/pipeline.mjs — orchestrate the content-hashed ingest pipeline.
- *
- * Flow: hash → de-dup → extract via docling/whisper → chunk → write to
- * .cx/ingest/<sha>/. Returns a record describing what was stored along
- * with any droppedInfo surfaced by the extractor.
- *
- * Idempotent: re-ingesting the same content (by sha256) returns the
- * existing record without re-extracting.
- */
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import { extractDocumentTextAsync } from '../document-extract.mjs';
-import { hashFile, defaultIngestRoot, readRecord, writeRecord } from './store.mjs';
-import { chunkMarkdown } from './chunker.mjs';
-
-export async function ingestFile(filePath, { cwd = process.cwd(), force = false } = {}) {
-  const absPath = path.resolve(filePath);
-  const stat = await fs.stat(absPath);
-  if (!stat.isFile()) throw new Error(`not a regular file: ${absPath}`);
-
-  const root = defaultIngestRoot(cwd);
-  const sha256 = await hashFile(absPath);
-
-  if (!force) {
-    const existing = await readRecord(root, sha256);
-    if (existing) return { ...existing, status: 'cached' };
-  }
-
-  const extracted = await extractDocumentTextAsync(absPath);
-  const markdown = extracted.markdown ?? extracted.text ?? '';
-  const chunks = chunkMarkdown(markdown);
-
-  const source = {
-    sourcePath: absPath,
-    fileName: path.basename(absPath),
-    extension: extracted.extension,
-    bytes: stat.size,
-    sha256,
-    ingestedAt: new Date().toISOString(),
-  };
-
-  const meta = {
-    extractionMethod: extracted.extractionMethod,
-    extractorMetadata: extracted.metadata ?? null,
-    droppedInfo: extracted.droppedInfo ?? [],
-    chunkCount: chunks.length,
-    chunkChars: chunks.reduce((a, c) => a + c.chars, 0),
-  };
-
-  const record = await writeRecord(root, { sha256, source, meta, markdown });
-  return { ...record, status: 'ingested', droppedInfo: meta.droppedInfo, chunks };
-}

package/lib/ingest/store.mjs

@@ -1,82 +0,0 @@
-/**
- * lib/ingest/store.mjs — content-addressed ingest store at .cx/ingest/<hash>/.
- *
- * Each ingested document is stored under its SHA-256 content hash:
- *   .cx/ingest/<sha256>/source.json   — original path, size, sha256, ingestedAt
- *   .cx/ingest/<sha256>/markdown.md   — extracted markdown body
- *   .cx/ingest/<sha256>/meta.json     — extractor metadata, droppedInfo, chunks
- *
- * Idempotent: re-ingesting a file with the same content is a no-op (returns
- * the existing record). Re-running ingestion is the safe default.
- *
- * 2026-06 best practice notes:
- *   - SHA-256 is still the universal content-addressing primitive. BLAKE3 is
- *     ~2.3× faster on M-series and worth considering at multi-GB scale,
- *     but is not the industry default and adds an unfamiliar dependency.
- *     Revisit if ingest throughput becomes a measured bottleneck.
- *   - Storing markdown body separately from metadata keeps the embedding
- *     and search paths simple: they only read markdown.md and look up
- *     provenance from meta.json on demand.
- */
-import { createHash } from 'node:crypto';
-import { createReadStream } from 'node:fs';
-import fs from 'node:fs/promises';
-import path from 'node:path';
-
-export function defaultIngestRoot(cwd = process.cwd()) {
-  return path.join(cwd, '.cx', 'ingest');
-}
-
-export async function hashFile(filePath) {
-  return new Promise((resolveHash, reject) => {
-    const hash = createHash('sha256');
-    const stream = createReadStream(filePath);
-    stream.on('data', (chunk) => hash.update(chunk));
-    stream.on('error', reject);
-    stream.on('end', () => resolveHash(hash.digest('hex')));
-  });
-}
-
-function recordDirFor(root, sha256) {
-  return path.join(root, sha256);
-}
-
-export async function readRecord(root, sha256) {
-  const dir = recordDirFor(root, sha256);
-  try {
-    const [source, meta, markdown] = await Promise.all([
-      fs.readFile(path.join(dir, 'source.json'), 'utf8').then(JSON.parse),
-      fs.readFile(path.join(dir, 'meta.json'), 'utf8').then(JSON.parse),
-      fs.readFile(path.join(dir, 'markdown.md'), 'utf8'),
-    ]);
-    return { sha256, source, meta, markdown, dir };
-  } catch {
-    return null;
-  }
-}
-
-export async function writeRecord(root, { sha256, source, meta, markdown }) {
-  const dir = recordDirFor(root, sha256);
-  await fs.mkdir(dir, { recursive: true });
-  await Promise.all([
-    fs.writeFile(path.join(dir, 'source.json'), JSON.stringify(source, null, 2) + '\n', 'utf8'),
-    fs.writeFile(path.join(dir, 'meta.json'), JSON.stringify(meta, null, 2) + '\n', 'utf8'),
-    fs.writeFile(path.join(dir, 'markdown.md'), markdown, 'utf8'),
-  ]);
-  return { sha256, dir, source, meta, markdown };
-}
-
-export async function listRecords(root) {
-  let entries;
-  try { entries = await fs.readdir(root, { withFileTypes: true }); }
-  catch { return []; }
-  const records = [];
-  for (const entry of entries) {
-    if (!entry.isDirectory()) continue;
-    const sha = entry.name;
-    if (!/^[a-f0-9]{64}$/.test(sha)) continue;
-    const record = await readRecord(root, sha);
-    if (record) records.push(record);
-  }
-  return records;
-}

package/lib/mode-commands.mjs

@@ -1,122 +0,0 @@
-/**
- * lib/mode-commands.mjs — Standardized command contracts across deployment modes.
- *
- * Provides mode-aware command execution for intake, memory, and workflow operations.
- * Routes to file-based implementations in solo mode, Postgres in team/enterprise mode.
- */
-
-// lib/mode-commands.mjs
-// Standardized command contracts across deployment modes
-
-import { spawnSync } from 'node:child_process';
-import { existsSync, readFileSync } from 'node:fs';
-import { basename, join } from 'node:path';
-
-export const MODE_COMMANDS = {
-  // Intake queue operations
-  intake: {
-    list: async (options = {}) => {
-      const mode = getDeploymentMode();
-      
-      if (mode === 'solo') {
-        // File-based intake
-        const result = spawnSync('find', ['.cx/intake/pending', '-name', '*.json', '-type', 'f'], 
-          { encoding: 'utf8', cwd: options.cwd });
-        return parseIntakeFiles(result.stdout);
-      } else {
-        // Postgres-based intake
-        return queryPostgresIntake(options);
-      }
-    },
-    
-    show: async (id, options = {}) => {
-      const mode = getDeploymentMode();
-      
-      if (mode === 'solo') {
-        return readIntakeFile(id, options);
-      } else {
-        return queryPostgresIntakeItem(id, options);
-      }
-    },
-  },
-  
-  // Storage operations
-  storage: {
-    sync: async (options = {}) => {
-      const mode = getDeploymentMode();
-      
-      // Same interface, different implementation
-      if (mode === 'solo') {
-        return syncToFileStorage(options);
-      } else {
-        return syncToPostgresStorage(options);
-      }
-    },
-    
-    query: async (criteria, options = {}) => {
-      // Unified query interface
-      return queryStorage(criteria, options);
-    },
-  },
-};
-
-function getDeploymentMode() {
-  return process.env.CONSTRUCT_DEPLOYMENT_MODE || 'solo';
-}
-
-function parseIntakeFiles(stdout) {
-  return stdout.split('\n')
-    .filter(Boolean)
-    .map(f => ({ id: basename(f, '.json'), source: 'file' }));
-}
-
-function readIntakeFile(id, options) {
-  const filePath = join(options.cwd || process.cwd(), '.cx/intake/pending', `${id}.json`);
-  if (!existsSync(filePath)) {
-    return { error: `Intake item ${id} not found` };
-  }
-  return JSON.parse(readFileSync(filePath, 'utf8'));
-}
-
-async function queryPostgresIntake(options) {
-  // Would use SQL client
-  return { items: [], mode: 'postgres' };
-}
-
-async function queryPostgresIntakeItem(id, options) {
-  return { id, mode: 'postgres' };
-}
-
-async function syncToFileStorage(options) {
-  return { 
-    success: true, 
-    mode: 'solo',
-    backend: 'file',
-  };
-}
-
-async function syncToPostgresStorage(options) {
-  return {
-    success: true,
-    mode: 'team',
-    backend: 'postgres',
-  };
-}
-
-async function queryStorage(criteria, options) {
-  // Unified query - works across modes
-  return {
-    results: [],
-    mode: getDeploymentMode(),
-  };
-}
-
-export function getCommandInterface(command, operation) {
-  const cmd = MODE_COMMANDS[command];
-  if (!cmd) return null;
-  
-  const fn = cmd[operation];
-  if (!fn) return null;
-  
-  return fn;
-}

package/lib/policy/unified-gates.mjs

@@ -1,96 +0,0 @@
-// lib/policy/unified-gates.mjs
-// Single policy declaration for all enforcement layers
-
-export const POLICIES = {
-  commentStyle: {
-    id: 'comment-style',
-    description: 'Comments must follow project standards',
-    layers: ['write', 'commit', 'ci'],
-    bypass: 'CONSTRUCT_SKIP_COMMENT_STYLE',
-    critical: false,
-  },
-  docCoupling: {
-    id: 'doc-coupling',
-    description: 'Code changes require documentation updates',
-    layers: ['commit', 'push'],
-    bypass: 'CONSTRUCT_SKIP_DOCS',
-    critical: false,
-  },
-  secretScan: {
-    id: 'secret-scan',
-    description: 'No secrets in committed code',
-    layers: ['commit', 'push', 'ci'],
-    bypass: 'CONSTRUCT_SKIP_SECRET_SCAN',
-    critical: true,
-  },
-  ciStatus: {
-    id: 'ci-status',
-    description: 'CI must be green before push',
-    layers: ['push', 'session-end'],
-    bypass: 'CONSTRUCT_SKIP_CI_CHECK',
-    critical: true,
-  },
-};
-
-export class UnifiedGateEngine {
-  constructor(options = {}) {
-    this.env = options.env || process.env;
-  }
-  
-  isBypassed(policyId) {
-    const policy = POLICIES[policyId];
-    if (!policy) return { bypassed: false };
-    
-    if (policy.bypass && this.env[policy.bypass] === '1') {
-      return { bypassed: true, envVar: policy.bypass };
-    }
-    
-    if (this.env.CONSTRUCT_SKIP_ALL_GATES === '1') {
-      return { bypassed: true, envVar: 'CONSTRUCT_SKIP_ALL_GATES' };
-    }
-    
-    return { bypassed: false };
-  }
-  
-  async evaluateLayer(layer, context = {}) {
-    const results = [];
-    
-    for (const [id, policy] of Object.entries(POLICIES)) {
-      if (policy.layers.includes(layer)) {
-        const bypass = this.isBypassed(id);
-        results.push({
-          policy: id,
-          passed: bypass.bypassed || true,
-          bypassed: bypass.bypassed,
-          bypassEnvVar: bypass.envVar,
-          critical: policy.critical,
-        });
-      }
-    }
-    
-    const failed = results.filter(r => !r.passed && r.critical);
-    
-    return {
-      layer,
-      passed: failed.length === 0,
-      canProceed: failed.length === 0,
-      results,
-      summary: `${results.filter(r => r.passed).length}/${results.length} passed`,
-    };
-  }
-}
-
-export async function checkGates(layer, context = {}, options = {}) {
-  const engine = new UnifiedGateEngine(options);
-  return await engine.evaluateLayer(layer, context);
-}
-
-export function listPolicies() {
-  return Object.entries(POLICIES).map(([id, policy]) => ({
-    id,
-    description: policy.description,
-    layers: policy.layers,
-    bypass: policy.bypass,
-    critical: policy.critical,
-  }));
-}

package/lib/profiles/validate-custom.mjs

@@ -1,114 +0,0 @@
-/**
- * lib/profiles/validate-custom.mjs — Schema validator for user-defined profiles.
- *
- * Custom profiles live at `<project>/.cx/profile.json` with `custom: true`.
- * They are validated on `construct sync` and via the pre-push gate. Rejected
- * profiles never get loaded; resolveActiveProfile falls back to the default
- * so a malformed escape-hatch file never breaks a project.
- *
- * Hard limits the validator enforces (kept in sync with schemas/profile.schema.json
- * and scripts/lint-profiles.mjs):
- *   - max 12 stages
- *   - max 24 intake types
- *   - max 80 roles per profile
- *   - max 12 departments per profile
- *   - max 20 roles per department
- *   - classificationTable path must stay inside .cx/ (no repo escapes)
- *
- * Rationale for each cap lives in docs/concepts/persona-research.md.
- */
-import fs from 'node:fs';
-import path from 'node:path';
-
-const MAX_STAGES = 12;
-const MAX_INTAKE_TYPES = 24;
-const MAX_ROLES = 80;
-const MAX_DEPARTMENTS = 12;
-const MAX_ROLES_PER_DEPARTMENT = 20;
-
-/**
- * @returns {string[]} array of error strings; empty means valid.
- */
-export function validateCustomProfile(profile, { cwd } = {}) {
-  const errors = [];
-  if (!profile || typeof profile !== 'object') return ['profile is not an object'];
-  if (profile.custom !== true) errors.push('custom profiles must set custom: true');
-
-  if (!profile.id || !/^[a-z][a-z0-9-]{1,30}$/.test(profile.id)) {
-    errors.push('id must match ^[a-z][a-z0-9-]{1,30}$');
-  }
-  if (!profile.displayName || typeof profile.displayName !== 'string') {
-    errors.push('displayName is required');
-  }
-  if (!Array.isArray(profile.roles) || profile.roles.length === 0) {
-    errors.push('roles must be a non-empty array');
-  } else if (profile.roles.length > MAX_ROLES) {
-    errors.push(`roles exceeds max of ${MAX_ROLES}`);
-  }
-  if (!profile.intake || typeof profile.intake !== 'object') {
-    errors.push('intake is required');
-  } else {
-    if (!Array.isArray(profile.intake.types) || profile.intake.types.length === 0) {
-      errors.push('intake.types must be a non-empty array');
-    } else if (profile.intake.types.length > MAX_INTAKE_TYPES) {
-      errors.push(`intake.types exceeds max of ${MAX_INTAKE_TYPES}`);
-    }
-    if (!Array.isArray(profile.intake.stages) || profile.intake.stages.length === 0) {
-      errors.push('intake.stages must be a non-empty array');
-    } else if (profile.intake.stages.length > MAX_STAGES) {
-      errors.push(`intake.stages exceeds max of ${MAX_STAGES}`);
-    }
-    if (typeof profile.intake.classificationTable === 'string') {
-      const t = profile.intake.classificationTable;
-      if (path.isAbsolute(t)) {
-        errors.push('intake.classificationTable must be a relative path');
-      } else if (cwd && !t.startsWith('.cx/')) {
-        errors.push('intake.classificationTable must live under .cx/ for custom profiles');
-      }
-    }
-  }
-  if (profile.departments !== undefined) {
-    if (!Array.isArray(profile.departments)) {
-      errors.push('departments must be an array');
-    } else {
-      if (profile.departments.length > MAX_DEPARTMENTS) {
-        errors.push(`departments exceeds max of ${MAX_DEPARTMENTS}`);
-      }
-      for (const [i, dept] of profile.departments.entries()) {
-        if (!dept || typeof dept !== 'object') {
-          errors.push(`departments[${i}] must be an object`);
-          continue;
-        }
-        if (!dept.id || !/^[a-z][a-z0-9-]{1,40}$/.test(dept.id)) {
-          errors.push(`departments[${i}].id is missing or malformed`);
-        }
-        if (!dept.charter || dept.charter.length < 20) {
-          errors.push(`departments[${i}].charter must be at least 20 chars (mission statement, not a label)`);
-        }
-        if (!Array.isArray(dept.roles) || dept.roles.length === 0) {
-          errors.push(`departments[${i}].roles must be a non-empty array`);
-        } else if (dept.roles.length > MAX_ROLES_PER_DEPARTMENT) {
-          errors.push(`departments[${i}].roles exceeds max of ${MAX_ROLES_PER_DEPARTMENT}`);
-        }
-      }
-    }
-  }
-  return errors;
-}
-
-/**
- * Read and validate `<cwd>/.cx/profile.json`. Returns:
- *   { status: 'absent' }                 if the file does not exist
- *   { status: 'invalid', errors: [...] } if validation failed
- *   { status: 'ok', profile }            if the file is a valid custom profile
- */
-export function validateCustomProfileFile(cwd) {
-  const p = path.join(cwd, '.cx', 'profile.json');
-  if (!fs.existsSync(p)) return { status: 'absent' };
-  let raw;
-  try { raw = JSON.parse(fs.readFileSync(p, 'utf8')); }
-  catch (err) { return { status: 'invalid', errors: [`malformed JSON: ${err.message}`] }; }
-  const errors = validateCustomProfile(raw, { cwd });
-  if (errors.length > 0) return { status: 'invalid', errors };
-  return { status: 'ok', profile: raw };
-}