@dotdotgod/context 0.2.25 → 0.2.26

package/README.md

@@ -36,6 +36,8 @@ Each command has one **10 MiB combined capture ceiling** shared by stdout and st
 
 `batch_execute` accepts up to 100 commands, uses concurrency from 1 through 8, and preserves input order. Execution and explicit shell mode remain write-capable operations subject to the host's approval model.
 
+`execute`, `batch_execute`, and `execute_file` share an `inherit-filtered-v1` child-environment policy. It preserves compatibility-oriented inheritance while filtering `NODE_OPTIONS`, `PYTHONPATH`, `RUBYOPT`, `LD_PRELOAD`, and supported-platform `DYLD_*` variables. Callers may provide string overrides or remove inherited variables with `null`, but cannot restore reserved variables. Results report filtered names without values. This policy applies only to these dotdotgod tools and does not isolate ordinary inherited credentials.
+
 ## Local Storage And Retrieval
 
 Indexed content is stored in the project-local SQLite FTS5 database:
@@ -44,7 +46,7 @@ Indexed content is stored in the project-local SQLite FTS5 database:
 .dotdotgod/context/context.sqlite
 ```
 
-The store is local ignored state, not maintained project truth. Sources can use transient, session, or project scope and carry optional expiry metadata. Search applies scope, session, and source predicates before candidate fusion, then:
+The store is local ignored state, not maintained project truth. Sources can use transient, session, or project scope and carry optional expiry metadata. Writable connections use WAL and a 1-second busy timeout. Source replacement, expiry, and purge update source and FTS rows transactionally. Existing compatible schemas reopen without migration; incompatible schemas fail before automatic repair. Search applies scope, session, and source predicates before candidate fusion, then:
 
 1. retrieves bounded Porter-tokenized FTS5 candidates;
 2. retrieves bounded label/path candidates;
@@ -56,7 +58,17 @@ This is bounded indexing and retrieval, not LLM summarization. Search does not a
 
 ## Structural Ingestion
 
-Local `index` accepts one regular text file up to **25 MiB**. Directory ingestion is not supported.
+Local `index` accepts one project-contained regular text file or directory. A file is limited to **25 MiB**. Directory defaults allow at most depth 16, 10,000 visited entries, 1,000 indexed files, 25 MiB per file, and 100 MiB in aggregate; callers may configure the traversal limits explicitly.
+
+Directory ingestion:
+
+- walks paths in deterministic lexical order and returns metadata-only indexed, skipped, and failed entries;
+- accepts explicit `includeExtensions` and `excludePaths` filters but does not interpret `.gitignore` files;
+- skips directory and file symlinks by default; optional file-symlink following remains limited to targets inside the project;
+- rejects special files and verifies file identity through a bounded descriptor read before indexing;
+- stops further traversal/indexing on cancellation while reporting already committed sources.
+
+Content chunking then applies these rules:
 
 - Markdown is split around headings, prose blocks, and fenced blocks while retaining heading and fence metadata.
 - JSON is parsed into deterministic key-path chunks; invalid JSON falls back to bounded text chunks.
@@ -91,7 +103,7 @@ The application-level fetch policy:
 - supports identity, gzip, deflate, and Brotli encodings and rejects unsupported encodings;
 - accepts bounded text-oriented MIME types and fails closed for unsupported types or HTTP statuses.
 
-HTML is handled as bounded untrusted text; the package does not claim rich HTML extraction. These controls provide application-level URL and address validation, not a network namespace or process sandbox.
+Accepted HTML is normalized by a bounded `html-v1` extractor. It validates supported UTF-8/US-ASCII declarations, removes scripts, styles, noscript and hidden/non-content elements, and preserves basic titles, headings, paragraphs, lists, tables, code, and link text. Link metadata is separately bounded. The extractor does not invoke a browser, execute JavaScript, load subresources, follow links, or provide standards-complete HTML rendering. HTML remains `external-untrusted`; normalization is not rendering sanitization or a prompt-injection guarantee. These controls provide application-level URL and address validation, not a network namespace or process sandbox.
 
 ## Diagnostics And Deletion
 
@@ -100,7 +112,7 @@ HTML is handled as bounded untrusted text; the package does not claim rich HTML
 - the minimum Node.js runtime;
 - project and storage-path readiness;
 - SQLite FTS5 with the Porter tokenizer;
-- compatibility of an existing context database schema;
+- compatibility of an existing context database schema without migration or repair;
 - current store statistics when supplied;
 - the configured strict fetch policy.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dotdotgod/context",
-  "version": "0.2.25",
+  "version": "0.2.26",
   "description": "Local-first MCP execution, indexing, and retrieval runtime for dotdotgod adapters.",
   "type": "module",
   "license": "Elastic-2.0",
@@ -28,7 +28,7 @@
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.25.2",
     "zod": "^3.25.76",
-    "@dotdotgod/cli": "0.2.25"
+    "@dotdotgod/cli": "0.2.26"
   },
   "engines": {
     "node": ">=22.5.0"

package/src/content.mjs

@@ -1,27 +1,79 @@
 import { readFileSync, statSync } from 'node:fs';
+import { readManifestFile, walkDirectoryManifest } from './directory-ingestion.mjs';
+import { normalizeHtml } from './html-normalize.mjs';
 import { resolveExistingWithinRoot } from './paths.mjs';
 import { safeFetch } from './safe-fetch.mjs';
 
 const MAX_FILE_BYTES = 25 * 1024 * 1024;
 const MAX_FETCH_BYTES = 10 * 1024 * 1024;
 
-export function indexFile(store, input, sessionId) {
-  const path = resolveExistingWithinRoot(input.root || process.cwd(), input.path);
-  const stat = statSync(path);
-  if (!stat.isFile()) throw new Error('Only regular files are supported in the first release.');
+function indexRegularFile(store, input, sessionId, path, stat, label = input.source ?? path, bytes) {
   const maxBytes = Math.min(input.maxBytes ?? MAX_FILE_BYTES, MAX_FILE_BYTES);
   if (stat.size > maxBytes) throw new Error(`File exceeds maximum size: ${stat.size} bytes`);
+  const content = bytes ?? readFileSync(path);
+  if (content.length > maxBytes) throw new Error(`File exceeds maximum size: ${content.length} bytes`);
   return store.index({
     scope: input.scope ?? 'session',
     sessionId,
-    label: input.source ?? path,
+    label,
     kind: 'file',
-    text: readFileSync(path, 'utf8'),
+    text: content.toString('utf8'),
     metadata: { path, size: stat.size },
     ttlMs: input.ttlMs ?? null,
   });
 }
 
+function indexDirectory(store, input, sessionId, signal) {
+  const manifest = walkDirectoryManifest({
+    root: input.root || process.cwd(),
+    path: input.path,
+    includeExtensions: input.includeExtensions,
+    excludePaths: input.excludePaths,
+    followFileSymlinks: input.followFileSymlinks,
+    maxDepth: input.maxDepth,
+    maxVisitedEntries: input.maxVisitedEntries,
+    maxFiles: input.maxFiles,
+    maxFileBytes: Math.min(input.maxBytes ?? MAX_FILE_BYTES, MAX_FILE_BYTES),
+    maxAggregateBytes: input.maxAggregateBytes,
+    signal,
+  });
+  const indexed = [];
+  const failed = [...manifest.failed];
+  const aggregateLimit = input.maxAggregateBytes ?? 100 * 1024 * 1024;
+  let indexedBytes = 0;
+  for (const entry of manifest.files) {
+    if (signal?.aborted) break;
+    try {
+      const remaining = aggregateLimit - indexedBytes;
+      if (remaining < 0) throw new Error('Directory aggregate byte limit exceeded.');
+      const bytes = readManifestFile(entry, manifest.root, Math.min(input.maxBytes ?? MAX_FILE_BYTES, MAX_FILE_BYTES, remaining));
+      const label = input.source ? `${input.source}/${entry.path}` : entry.absolutePath;
+      indexed.push({ path: entry.path, ...indexRegularFile(store, input, sessionId, entry.absolutePath, entry, label, bytes) });
+      indexedBytes += bytes.length;
+    } catch (error) {
+      failed.push({ path: entry.path, reason: 'index-failed', error: error instanceof Error ? error.message : String(error) });
+    }
+  }
+  return {
+    directory: manifest.directory,
+    indexed,
+    skipped: manifest.skipped,
+    failed,
+    aborted: manifest.aborted || signal?.aborted === true,
+    truncated: manifest.truncated,
+    visitedEntries: manifest.visitedEntries,
+    aggregateBytes: indexedBytes,
+  };
+}
+
+export function indexFile(store, input, sessionId, signal) {
+  const path = resolveExistingWithinRoot(input.root || process.cwd(), input.path);
+  const stat = statSync(path);
+  if (stat.isDirectory()) return indexDirectory(store, input, sessionId, signal);
+  if (!stat.isFile()) throw new Error('Only regular files and directories are supported.');
+  return indexRegularFile(store, input, sessionId, path, stat);
+}
+
 export async function fetchAndIndex(store, input, sessionId, signal) {
   const maxBytes = Math.min(input.maxBytes ?? MAX_FETCH_BYTES, MAX_FETCH_BYTES);
   const fetched = await safeFetch(input.url, {
@@ -30,12 +82,19 @@ export async function fetchAndIndex(store, input, sessionId, signal) {
     maxBytes,
     maxWireBytes: maxBytes,
   });
+  const isHtml = /^(?:text\/html|application\/xhtml\+xml)(?:;|$)/iu.test(fetched.contentType);
+  const normalized = isHtml ? normalizeHtml(fetched.body, {
+    contentType: fetched.contentType,
+    maxInputBytes: maxBytes,
+    maxOutputBytes: maxBytes,
+  }) : null;
+  const text = normalized?.text ?? fetched.text;
   return store.index({
     scope: input.scope ?? 'project',
     sessionId,
-    label: input.source ?? fetched.url,
+    label: input.source ?? normalized?.title ?? fetched.url,
     kind: 'url',
-    text: fetched.text,
+    text,
     metadata: {
       url: fetched.url,
       originalUrl: fetched.originalUrl,
@@ -46,8 +105,20 @@ export async function fetchAndIndex(store, input, sessionId, signal) {
       status: fetched.status,
       bytes: fetched.bytes,
       wireLength: fetched.wireLength,
+      ...(normalized ? {
+        htmlTitle: normalized.title,
+        htmlInputBytes: normalized.inputBytes,
+        htmlOutputBytes: normalized.outputBytes,
+        htmlTruncated: normalized.truncated,
+        htmlFallbackReason: normalized.fallbackReason,
+      } : {}),
+    },
+    provenance: {
+      sourceType: 'fetched-url',
+      origin: fetched.url,
+      contentType: normalized ? 'text/markdown' : fetched.contentType,
+      extractor: normalized?.extractor,
     },
-    provenance: { sourceType: 'fetched-url', origin: fetched.url, contentType: fetched.contentType },
     ttlMs: input.ttlMs ?? 24 * 60 * 60 * 1000,
   });
 }

package/src/directory-ingestion.mjs

@@ -0,0 +1,182 @@
+import { closeSync, constants, fstatSync, lstatSync, openSync, readFileSync, readdirSync, realpathSync, statSync } from 'node:fs';
+import { isAbsolute, relative, resolve, sep } from 'node:path';
+
+const DEFAULT_LIMITS = Object.freeze({
+  maxDepth: 16,
+  maxVisitedEntries: 10_000,
+  maxFiles: 1_000,
+  maxFileBytes: 25 * 1024 * 1024,
+  maxAggregateBytes: 100 * 1024 * 1024,
+});
+
+function inside(base, target) {
+  const rel = relative(base, target);
+  return rel !== '..' && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
+}
+
+function relativePath(base, target) {
+  return relative(base, target).split(sep).join('/');
+}
+
+function boundedInteger(value, fallback, name) {
+  const selected = value ?? fallback;
+  if (!Number.isSafeInteger(selected) || selected < 0) throw new TypeError(`${name} must be a non-negative safe integer`);
+  return selected;
+}
+
+function normalizeOptions(options) {
+  const includeExtensions = options.includeExtensions?.map((value) => {
+    const extension = String(value).toLowerCase();
+    return extension.startsWith('.') ? extension : `.${extension}`;
+  });
+  const excludePaths = (options.excludePaths ?? []).map((value) => {
+    const normalized = String(value).replaceAll('\\', '/').replace(/^\.\//, '').replace(/\/$/, '');
+    if (!normalized || normalized === '..' || normalized.startsWith('../') || normalized.includes('/../')) {
+      throw new TypeError(`Invalid excluded path: ${value}`);
+    }
+    return normalized;
+  });
+  return {
+    includeExtensions: includeExtensions ? new Set(includeExtensions) : null,
+    excludePaths,
+    followFileSymlinks: options.followFileSymlinks === true,
+    maxDepth: boundedInteger(options.maxDepth, DEFAULT_LIMITS.maxDepth, 'maxDepth'),
+    maxVisitedEntries: boundedInteger(options.maxVisitedEntries, DEFAULT_LIMITS.maxVisitedEntries, 'maxVisitedEntries'),
+    maxFiles: boundedInteger(options.maxFiles, DEFAULT_LIMITS.maxFiles, 'maxFiles'),
+    maxFileBytes: boundedInteger(options.maxFileBytes, DEFAULT_LIMITS.maxFileBytes, 'maxFileBytes'),
+    maxAggregateBytes: boundedInteger(options.maxAggregateBytes, DEFAULT_LIMITS.maxAggregateBytes, 'maxAggregateBytes'),
+  };
+}
+
+function excluded(path, prefixes) {
+  return prefixes.some((prefix) => path === prefix || path.startsWith(`${prefix}/`));
+}
+
+function extensionAllowed(path, extensions) {
+  if (!extensions) return true;
+  const slash = path.lastIndexOf('/');
+  const dot = path.lastIndexOf('.');
+  return dot > slash && extensions.has(path.slice(dot).toLowerCase());
+}
+
+function snapshot(path, target, stat, symlink) {
+  return {
+    path,
+    absolutePath: target,
+    size: stat.size,
+    identity: { dev: stat.dev, ino: stat.ino, size: stat.size, mtimeMs: stat.mtimeMs },
+    symlink,
+  };
+}
+
+export function walkDirectoryManifest(options = {}) {
+  const root = realpathSync(resolve(options.root || process.cwd()));
+  const requested = resolve(root, options.path ?? '.');
+  if (!inside(root, requested)) throw new Error(`Directory escapes project root: ${options.path}`);
+  if (lstatSync(requested).isSymbolicLink()) throw new Error(`Directory symlinks are not followed: ${options.path ?? '.'}`);
+  const directory = realpathSync(requested);
+  if (!inside(root, directory)) throw new Error(`Directory escapes project root: ${options.path}`);
+  if (!statSync(directory).isDirectory()) throw new Error(`Not a directory: ${options.path ?? '.'}`);
+
+  const policy = normalizeOptions(options);
+  const files = [];
+  const skipped = [];
+  const failed = [];
+  let visitedEntries = 0;
+  let aggregateBytes = 0;
+  let aborted = false;
+  let truncated = false;
+
+  function skip(path, reason) { skipped.push({ path, reason }); }
+  function fail(path, reason) { failed.push({ path, reason }); }
+
+  function visit(current, depth) {
+    if (aborted || options.signal?.aborted) { aborted = true; return; }
+    let names;
+    try { names = readdirSync(current).sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); }
+    catch { fail(relativePath(root, current), 'directory-read-failed'); return; }
+
+    for (const name of names) {
+      if (options.signal?.aborted) { aborted = true; return; }
+      const absolutePath = resolve(current, name);
+      const path = relativePath(root, absolutePath);
+      if (visitedEntries >= policy.maxVisitedEntries) { skip(path, 'visited-entry-limit'); truncated = true; return; }
+      visitedEntries += 1;
+      if (excluded(path, policy.excludePaths)) { skip(path, 'excluded-path'); continue; }
+
+      let entry;
+      try { entry = lstatSync(absolutePath); }
+      catch { fail(path, 'entry-stat-failed'); continue; }
+
+      if (entry.isSymbolicLink()) {
+        let target;
+        let targetStat;
+        try {
+          target = realpathSync(absolutePath);
+          if (!inside(root, target)) { skip(path, 'symlink-outside-root'); continue; }
+          targetStat = statSync(target);
+        } catch { skip(path, 'symlink-unresolvable'); continue; }
+        if (targetStat.isDirectory()) { skip(path, 'directory-symlink'); continue; }
+        if (!policy.followFileSymlinks) { skip(path, 'file-symlink'); continue; }
+        if (!targetStat.isFile()) { skip(path, 'special-file'); continue; }
+        addFile(path, target, targetStat, true);
+        continue;
+      }
+
+      if (entry.isDirectory()) {
+        if (depth >= policy.maxDepth) { skip(path, 'depth-limit'); continue; }
+        visit(absolutePath, depth + 1);
+        if (aborted || truncated) return;
+        continue;
+      }
+      if (!entry.isFile()) { skip(path, 'special-file'); continue; }
+      addFile(path, absolutePath, entry, false);
+    }
+  }
+
+  function addFile(path, absolutePath, stat, symlink) {
+    if (!extensionAllowed(path, policy.includeExtensions)) { skip(path, 'extension-filter'); return; }
+    if (stat.size > policy.maxFileBytes) { skip(path, 'file-byte-limit'); return; }
+    if (files.length >= policy.maxFiles) { skip(path, 'file-count-limit'); return; }
+    if (aggregateBytes + stat.size > policy.maxAggregateBytes) { skip(path, 'aggregate-byte-limit'); return; }
+    files.push(snapshot(path, absolutePath, stat, symlink));
+    aggregateBytes += stat.size;
+  }
+
+  visit(directory, 0);
+  return { root, directory, files, skipped, failed, aborted, truncated, visitedEntries, aggregateBytes };
+}
+
+function matchesIdentity(stat, identity) {
+  return stat.dev === identity.dev && stat.ino === identity.ino && stat.size === identity.size && stat.mtimeMs === identity.mtimeMs;
+}
+
+export function verifyManifestFile(entry, root) {
+  const base = realpathSync(resolve(root));
+  const target = realpathSync(entry.absolutePath);
+  if (!inside(base, target)) throw new Error(`Manifest file escapes project root: ${entry.path}`);
+  const stat = statSync(target);
+  if (!stat.isFile()) throw new Error(`Manifest entry is not a regular file: ${entry.path}`);
+  if (!matchesIdentity(stat, entry.identity ?? {})) throw new Error(`Manifest file changed before indexing: ${entry.path}`);
+  return target;
+}
+
+export function readManifestFile(entry, root, maxBytes = DEFAULT_LIMITS.maxFileBytes) {
+  const target = verifyManifestFile(entry, root);
+  const noFollow = constants.O_NOFOLLOW ?? 0;
+  const descriptor = openSync(target, constants.O_RDONLY | noFollow);
+  try {
+    const before = fstatSync(descriptor);
+    if (!before.isFile() || !matchesIdentity(before, entry.identity ?? {})) throw new Error(`Manifest file changed before indexing: ${entry.path}`);
+    if (before.size > maxBytes) throw new Error(`Manifest file exceeds maximum size: ${entry.path}`);
+    const bytes = readFileSync(descriptor);
+    const after = fstatSync(descriptor);
+    if (!matchesIdentity(after, entry.identity ?? {}) || bytes.length !== before.size) throw new Error(`Manifest file changed while indexing: ${entry.path}`);
+    if (bytes.length > maxBytes) throw new Error(`Manifest file exceeds maximum size: ${entry.path}`);
+    return bytes;
+  } finally {
+    closeSync(descriptor);
+  }
+}
+
+export { DEFAULT_LIMITS as DIRECTORY_INGESTION_LIMITS };

package/src/doctor.mjs

@@ -92,11 +92,20 @@ function schemaCheck(dbPath) {
     const names = new Set(db.prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view')").all().map((row) => row.name));
     const missing = [...EXPECTED_TABLES].filter((name) => !names.has(name));
     if (missing.length > 0) return check('schema-compatibility', 'FAIL', 'Context database is missing required tables.', { dbPath, missing });
-    const sourceColumns = new Set(db.prepare('PRAGMA table_info(sources)').all().map((row) => row.name));
-    const requiredColumns = ['id', 'scope', 'session_id', 'label', 'kind', 'metadata', 'created_at', 'expires_at'];
-    const missingColumns = requiredColumns.filter((name) => !sourceColumns.has(name));
-    if (missingColumns.length > 0) return check('schema-compatibility', 'FAIL', 'Sources table is incompatible.', { dbPath, missingColumns });
-    return check('schema-compatibility', 'OK', 'Existing context database has the required schema.', { dbPath });
+    const sourceRows = db.prepare('PRAGMA table_info(sources)').all();
+    const chunkRows = db.prepare('PRAGMA table_info(chunks)').all();
+    const requiredSources = ['id', 'scope', 'session_id', 'label', 'kind', 'metadata', 'created_at', 'expires_at'];
+    const requiredChunks = ['source_id', 'scope', 'session_id', 'ordinal', 'body'];
+    const missingColumns = [...requiredSources.filter((name) => !sourceRows.some((row) => row.name === name)), ...requiredChunks.filter((name) => !chunkRows.some((row) => row.name === name))];
+    const sourceShape = sourceRows.length === requiredSources.length && sourceRows.every((row, index) => row.name === requiredSources[index])
+      && sourceRows[0].pk === 1
+      && sourceRows.every((row) => ['scope', 'label', 'kind', 'metadata', 'created_at'].includes(row.name) ? row.notnull === 1 : true);
+    const chunkShape = chunkRows.length === requiredChunks.length && chunkRows.every((row, index) => row.name === requiredChunks[index]);
+    const chunkSql = String(db.prepare("SELECT sql FROM sqlite_master WHERE name = 'chunks'").get()?.sql ?? '').toLowerCase();
+    if (missingColumns.length > 0 || !sourceShape || !chunkShape || !chunkSql.includes('using fts5') || !chunkSql.includes("tokenize='porter unicode61'")) {
+      return check('schema-compatibility', 'FAIL', 'Context database schema is incompatible.', { dbPath, schemaProfile: 'unknown', missingColumns });
+    }
+    return check('schema-compatibility', 'OK', 'Existing context database matches compatibility profile 1.', { dbPath, schemaProfile: 1 });
   } catch (error) {
     return check('schema-compatibility', 'FAIL', 'Context database could not be inspected read-only.', { dbPath, error: error.message });
   } finally {

package/src/environment-policy.mjs

@@ -0,0 +1,80 @@
+const COMMON_FILTERED_NAMES = new Set(['NODE_OPTIONS', 'PYTHONPATH', 'RUBYOPT']);
+const VALID_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
+function platformFamily(platform) {
+  if (platform === 'win32') return 'win32';
+  if (platform === 'darwin') return 'darwin';
+  return 'posix';
+}
+
+function canonicalName(name, family) {
+  return family === 'win32' ? name.toUpperCase() : name;
+}
+
+function validateName(name) {
+  if (!VALID_NAME.test(name)) throw new TypeError(`Invalid environment variable name: ${JSON.stringify(name)}`);
+}
+
+function isFilteredName(name, family) {
+  const comparable = family === 'win32' ? name.toUpperCase() : name;
+  if (COMMON_FILTERED_NAMES.has(comparable)) return true;
+  if (family === 'darwin') return comparable === 'LD_PRELOAD' || comparable.startsWith('DYLD_');
+  if (family === 'posix') return comparable === 'LD_PRELOAD';
+  return false;
+}
+
+function entriesOf(value, label) {
+  if (value === null || typeof value !== 'object' || Array.isArray(value)) {
+    throw new TypeError(`${label} must be an object`);
+  }
+  return Object.entries(value).sort(([left], [right]) => left.localeCompare(right, 'en'));
+}
+
+/**
+ * Compose a child-process environment without exposing filtered values.
+ * Override values may be strings or null; null explicitly deletes a variable.
+ */
+export function composeEnvironment({ inherited = {}, overrides = {}, platform = process.platform } = {}) {
+  const family = platformFamily(platform);
+  const env = {};
+  const filtered = new Set();
+
+  for (const [name, value] of entriesOf(inherited, 'inherited')) {
+    validateName(name);
+    if (typeof value !== 'string' && value !== undefined) {
+      throw new TypeError(`Inherited environment variable ${name} must be a string or undefined`);
+    }
+    if (value === undefined) continue;
+    const outputName = canonicalName(name, family);
+    if (isFilteredName(outputName, family)) {
+      filtered.add(outputName);
+      continue;
+    }
+    env[outputName] = value;
+  }
+
+  for (const [name, value] of entriesOf(overrides, 'overrides')) {
+    validateName(name);
+    const outputName = canonicalName(name, family);
+    if (isFilteredName(outputName, family)) {
+      throw new TypeError(`Environment variable ${name} is reserved by the execution policy`);
+    }
+    if (value === null) {
+      delete env[outputName];
+      continue;
+    }
+    if (typeof value !== 'string') {
+      throw new TypeError(`Environment override ${name} must be a string or null`);
+    }
+    env[outputName] = value;
+  }
+
+  return {
+    env,
+    policy: {
+      mode: 'inherit-filtered-v1',
+      platform: family,
+      filteredNames: [...filtered].sort((left, right) => left.localeCompare(right, 'en')),
+    },
+  };
+}

package/src/execute.mjs

@@ -4,6 +4,7 @@ import { join } from 'node:path';
 import { spawn } from 'node:child_process';
 import { Transform } from 'node:stream';
 import { resolveWithinRoot } from './paths.mjs';
+import { composeEnvironment } from './environment-policy.mjs';
 
 const DIRECT_LIMIT = 12_000;
 const MAX_DIRECT_RETURN = 1024 * 1024;
@@ -32,6 +33,7 @@ function killProcess(child) {
 }
 
 export async function executeCommand(input, options = {}) {
+  if (options.signal?.aborted) throw options.signal.reason ?? new Error('Execution aborted.');
   const spec = commandSpec(input);
   const cwd = resolveWithinRoot(options.root || process.cwd(), input.cwd || '.');
   const timeoutMs = Math.min(Math.max(1, input.timeoutMs ?? 120_000), 30 * 60_000);
@@ -45,6 +47,10 @@ export async function executeCommand(input, options = {}) {
   let aborted = false;
   let captureLimitExceeded = false;
   const captureLimitBytes = Math.min(HARD_LIMIT, Math.max(1024, options.captureLimitBytes ?? HARD_LIMIT));
+  const environment = composeEnvironment({
+    inherited: options.env ?? process.env,
+    overrides: input.env ?? {},
+  });
 
   try {
     const result = await new Promise((resolvePromise, reject) => {
@@ -53,7 +59,7 @@ export async function executeCommand(input, options = {}) {
         shell: spec.shell,
         detached: process.platform !== 'win32',
         stdio: ['ignore', 'pipe', 'pipe'],
-        env: options.env ?? process.env,
+        env: environment.env,
       });
       let capturedBytes = 0;
       const boundedCapture = () => new Transform({
@@ -99,6 +105,7 @@ export async function executeCommand(input, options = {}) {
       durationMs: Date.now() - startedAt,
       stdoutBytes: stdoutResult.bytes,
       stderrBytes: stderrResult.bytes,
+      environmentPolicy: environment.policy,
     };
     const mode = input.outputMode ?? 'auto';
     const shouldIndex = mode === 'indexed' || (mode === 'auto' && totalBytes > directLimit);
@@ -128,6 +135,7 @@ export async function executeCommand(input, options = {}) {
 
 export async function executeFile(input, options = {}) {
   const path = resolveWithinRoot(options.root || process.cwd(), input.path);
+  const selectorEnvironment = composeEnvironment({ inherited: options.env ?? process.env, overrides: input.env ?? {} }).env;
   const dir = mkdtempSync(join(tmpdir(), 'dotdotgod-context-file-'));
   try {
     let executable;
@@ -135,12 +143,12 @@ export async function executeFile(input, options = {}) {
     if (input.language === 'python') {
       const wrapper = join(dir, 'run.py');
       writeFileSync(wrapper, `from pathlib import Path\nFILE_CONTENT = Path(${JSON.stringify(path)}).read_text(encoding='utf-8', errors='replace')\n${input.code}\n`);
-      executable = process.env.PYTHON || 'python3';
+      executable = selectorEnvironment.PYTHON || 'python3';
       args = [wrapper];
     } else if (input.language === 'shell') {
       const wrapper = join(dir, 'run.sh');
       writeFileSync(wrapper, `FILE_PATH=${JSON.stringify(path)}\nexport FILE_PATH\n${input.code}\n`);
-      executable = process.env.SHELL || '/bin/sh';
+      executable = selectorEnvironment.SHELL || '/bin/sh';
       args = [wrapper];
     } else {
       const wrapper = join(dir, 'run.mjs');

package/src/html-normalize.mjs

@@ -0,0 +1,195 @@
+const DEFAULT_INPUT_BYTES = 10 * 1024 * 1024;
+const DEFAULT_OUTPUT_BYTES = 10 * 1024 * 1024;
+const EXTRACTOR = 'html-v1';
+const ACCEPTED_MIME = new Set(['text/html', 'application/xhtml+xml']);
+const ACCEPTED_CHARSET = new Set(['utf-8', 'utf8', 'us-ascii']);
+const MAX_LINKS = 100;
+const MAX_LINK_METADATA_BYTES = 64 * 1024;
+const BLOCK_TAGS = new Set(['address', 'article', 'aside', 'blockquote', 'br', 'dd', 'div', 'dl', 'dt', 'figcaption', 'figure', 'footer', 'header', 'hr', 'main', 'nav', 'p', 'section', 'tr']);
+const DROP_CONTENT = new Set(['script', 'style', 'noscript', 'template', 'svg', 'math', 'iframe', 'object', 'embed', 'form']);
+
+function positiveLimit(value, fallback) {
+  if (value === undefined) return fallback;
+  if (!Number.isFinite(value) || value <= 0) throw new TypeError('HTML byte limits must be positive finite numbers.');
+  return Math.floor(value);
+}
+
+function parseMediaType(value) {
+  const raw = String(value ?? 'text/html');
+  const [type, ...parameters] = raw.split(';');
+  const mimeType = type.trim().toLowerCase();
+  if (!ACCEPTED_MIME.has(mimeType)) throw new Error(`Unsupported HTML MIME type: ${mimeType || '(empty)'}`);
+  let charset = 'utf-8';
+  for (const parameter of parameters) {
+    const match = /^\s*charset\s*=\s*(?:"([^"]+)"|'([^']+)'|([^\s;]+))\s*$/iu.exec(parameter);
+    if (match) charset = (match[1] || match[2] || match[3]).toLowerCase();
+  }
+  if (!ACCEPTED_CHARSET.has(charset)) throw new Error(`Unsupported HTML charset: ${charset}`);
+  return { mimeType, charset: charset === 'utf8' ? 'utf-8' : charset };
+}
+
+function decodeEntities(value) {
+  const named = { amp: '&', apos: "'", gt: '>', lt: '<', nbsp: ' ', quot: '"' };
+  return value.replace(/&(#(?:x[0-9a-f]+|\d+)|[a-z]+);/giu, (whole, entity) => {
+    if (entity[0] !== '#') return named[entity.toLowerCase()] ?? whole;
+    const hexadecimal = entity[1]?.toLowerCase() === 'x';
+    const number = Number.parseInt(entity.slice(hexadecimal ? 2 : 1), hexadecimal ? 16 : 10);
+    if (!Number.isInteger(number) || number < 0 || number > 0x10ffff || (number >= 0xd800 && number <= 0xdfff)) return '\uFFFD';
+    return String.fromCodePoint(number);
+  });
+}
+
+function normalizeWhitespace(value) {
+  return value
+    .replace(/[\t\f\v ]+/gu, ' ')
+    .replace(/ *\n */gu, '\n')
+    .replace(/\n{3,}/gu, '\n\n')
+    .trim();
+}
+
+function boundedUtf8(value, maxBytes) {
+  if (Buffer.byteLength(value) <= maxBytes) return { value, truncated: false };
+  let output = '';
+  let bytes = 0;
+  for (const point of value) {
+    const size = Buffer.byteLength(point);
+    if (bytes + size > maxBytes) break;
+    output += point;
+    bytes += size;
+  }
+  return { value: output.trimEnd(), truncated: true };
+}
+
+function tagInfo(raw) {
+  const closing = /^<\s*\//u.test(raw);
+  const match = /^<\s*\/?\s*([a-z][a-z0-9:-]*)/iu.exec(raw);
+  if (!match) return null;
+  const name = match[1].toLowerCase();
+  return { name, closing, selfClosing: /\/\s*>$/u.test(raw), attributes: closing ? '' : raw.slice(match[0].length, -1) };
+}
+
+function hiddenElement(attributes) {
+  return /(?:^|\s)hidden(?:\s|=|$)/iu.test(attributes)
+    || /(?:^|\s)aria-hidden\s*=\s*(?:["']?true["']?)/iu.test(attributes)
+    || /(?:^|\s)type\s*=\s*(?:["']?hidden["']?)/iu.test(attributes)
+    || /(?:^|\s)style\s*=\s*(["'])[^"']*(?:display\s*:\s*none|visibility\s*:\s*hidden)[^"']*\1/iu.test(attributes);
+}
+
+function hrefFrom(attributes) {
+  const match = /(?:^|\s)href\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+))/iu.exec(attributes);
+  return decodeEntities(match ? (match[1] ?? match[2] ?? match[3] ?? '') : '').trim();
+}
+
+export function normalizeHtml(value, options = {}) {
+  const maxInputBytes = positiveLimit(options.maxInputBytes, DEFAULT_INPUT_BYTES);
+  const maxOutputBytes = positiveLimit(options.maxOutputBytes, DEFAULT_OUTPUT_BYTES);
+  const { mimeType, charset } = parseMediaType(options.contentType);
+  const input = Buffer.isBuffer(value) ? value : Buffer.from(String(value ?? ''), 'utf8');
+  const inputBytes = input.length;
+  if (inputBytes > maxInputBytes) throw new Error(`HTML input exceeds maximum size: ${inputBytes} bytes`);
+  if (charset === 'us-ascii' && input.some((byte) => byte > 0x7f)) throw new Error('HTML body contains bytes outside the declared US-ASCII charset.');
+  let html;
+  try { html = new TextDecoder('utf-8', { fatal: true }).decode(input); }
+  catch { throw new Error('HTML body is not valid UTF-8.'); }
+  const declaredCharsets = [...html.matchAll(/<meta\b[^>]*charset\s*=\s*["']?([^\s"'/>;]+)/giu)].map((match) => match[1].toLowerCase().replace('utf8', 'utf-8'));
+  for (const match of html.matchAll(/<meta\b(?=[^>]*http-equiv\s*=\s*["']?content-type["']?)[^>]*content\s*=\s*(?:"[^"]*charset\s*=\s*([^\s";]+)[^"]*"|'[^']*charset\s*=\s*([^\s';]+)[^']*'|[^>]*charset\s*=\s*([^\s;>]+))[^>]*>/giu)) {
+    declaredCharsets.push((match[1] || match[2] || match[3]).toLowerCase().replace('utf8', 'utf-8'));
+  }
+  if (declaredCharsets.some((declared) => !ACCEPTED_CHARSET.has(declared) || declared !== charset)) throw new Error('HTML document charset conflicts with the response charset.');
+  html = html.replace(/\0/gu, '\uFFFD').replace(/\r\n?/gu, '\n');
+  const lower = html.toLowerCase();
+  const output = [];
+  const headings = [];
+  const links = [];
+  const skip = [];
+  const anchors = [];
+  let title = null;
+  let titleStart = -1;
+  let preDepth = 0;
+  let malformed = false;
+  let cursor = 0;
+
+  const append = (text) => { if (!skip.length && text) output.push(text); };
+  while (cursor < html.length) {
+    const open = html.indexOf('<', cursor);
+    if (open < 0) {
+      append(decodeEntities(html.slice(cursor)));
+      break;
+    }
+    append(decodeEntities(html.slice(cursor, open)));
+    if (lower.startsWith('<!--', open)) {
+      const end = lower.indexOf('-->', open + 4);
+      if (end < 0) { malformed = true; break; }
+      cursor = end + 3;
+      continue;
+    }
+    const close = html.indexOf('>', open + 1);
+    if (close < 0) { malformed = true; break; }
+    const raw = html.slice(open, close + 1);
+    if (/^<!|^<\?/u.test(raw)) { cursor = close + 1; continue; }
+    const tag = tagInfo(raw);
+    if (!tag) { malformed = true; cursor = close + 1; continue; }
+
+    if (skip.length) {
+      if (!tag.closing && !tag.selfClosing && tag.name === skip.at(-1)) skip.push(tag.name);
+      else if (tag.closing && tag.name === skip.at(-1)) skip.pop();
+      cursor = close + 1;
+      continue;
+    }
+    if (!tag.closing && (DROP_CONTENT.has(tag.name) || hiddenElement(tag.attributes))) {
+      if (!tag.selfClosing) skip.push(tag.name);
+      cursor = close + 1;
+      continue;
+    }
+    if (tag.name === 'title') {
+      if (!tag.closing) titleStart = output.length;
+      else if (titleStart >= 0) {
+        title = normalizeWhitespace(output.splice(titleStart).join('')) || null;
+        titleStart = -1;
+      }
+    } else if (/^h[1-6]$/u.test(tag.name)) {
+      if (!tag.closing) {
+        append(`\n${'#'.repeat(Number(tag.name[1]))} `);
+        headings.push(tag.name);
+      } else if (headings.at(-1) === tag.name) {
+        headings.pop();
+        append('\n');
+      }
+    } else if (tag.name === 'li') append(tag.closing ? '\n' : '\n- ');
+    else if (tag.name === 'pre') {
+      if (tag.closing) { preDepth = Math.max(0, preDepth - 1); append('\n```\n'); }
+      else { append('\n```\n'); preDepth += 1; }
+    } else if (tag.name === 'code' && !tag.closing) append(preDepth ? '' : '`');
+    else if (tag.name === 'code' && tag.closing) append(preDepth ? '' : '`');
+    else if (tag.name === 'a') {
+      if (!tag.closing) anchors.push({ start: output.length, href: hrefFrom(tag.attributes) });
+      else {
+        const anchor = anchors.pop();
+        if (anchor) {
+          const text = normalizeWhitespace(output.slice(anchor.start).join(''));
+          if (text && anchor.href && links.length < MAX_LINKS) {
+            const link = { text: boundedUtf8(text, 500).value, href: boundedUtf8(anchor.href, 2048).value };
+            if (Buffer.byteLength(JSON.stringify([...links, link])) <= MAX_LINK_METADATA_BYTES) links.push(link);
+          }
+        }
+      }
+    } else if (tag.name === 'td' || tag.name === 'th') append(tag.closing ? ' | ' : '| ');
+    else if (BLOCK_TAGS.has(tag.name)) append(tag.name === 'br' || tag.closing ? '\n' : '\n');
+    cursor = close + 1;
+  }
+  if (skip.length || titleStart >= 0 || anchors.length || headings.length) malformed = true;
+  const normalized = normalizeWhitespace(output.join(''));
+  const bounded = boundedUtf8(normalized, maxOutputBytes);
+  return {
+    text: bounded.value,
+    title,
+    links,
+    extractor: EXTRACTOR,
+    fallbackReason: malformed ? 'malformed-html' : null,
+    mimeType,
+    charset,
+    inputBytes,
+    outputBytes: Buffer.byteLength(bounded.value),
+    truncated: bounded.truncated,
+  };
+}

package/src/index.d.ts

@@ -1,20 +1,37 @@
+export type ContextScope = 'transient' | 'session' | 'project';
+export type DirectoryReason = 'excluded-path' | 'extension-filter' | 'file-byte-limit' | 'file-count-limit' | 'aggregate-byte-limit' | 'visited-entry-limit' | 'depth-limit' | 'file-symlink' | 'directory-symlink' | 'symlink-outside-root' | 'symlink-unresolvable' | 'special-file' | 'directory-read-failed' | 'entry-stat-failed' | 'index-failed';
+export interface EnvironmentPolicy { mode: 'inherit-filtered-v1'; platform: 'win32' | 'darwin' | 'posix'; filteredNames: string[] }
+export interface DirectoryManifestEntry { path: string; absolutePath: string; size: number; identity: { dev: number; ino: number; size: number; mtimeMs: number }; symlink: boolean }
+export interface DirectoryManifest { root: string; directory: string; files: DirectoryManifestEntry[]; skipped: Array<{ path: string; reason: DirectoryReason }>; failed: Array<{ path: string; reason: DirectoryReason; error?: string }>; aborted: boolean; truncated: boolean; visitedEntries: number; aggregateBytes: number }
+export interface HtmlNormalization { text: string; title: string | null; links: Array<{ text: string; href: string }>; extractor: 'html-v1'; fallbackReason: 'malformed-html' | null; mimeType: 'text/html' | 'application/xhtml+xml'; charset: 'utf-8' | 'us-ascii'; inputBytes: number; outputBytes: number; truncated: boolean }
+export interface ExecutionInput { label?: string; command?: string; executable?: string; args?: string[]; shell?: boolean; cwd?: string; timeoutMs?: number; outputLimit?: number; directLimit?: number; outputMode?: 'auto' | 'direct' | 'indexed' | 'discard'; scope?: ContextScope; ttlMs?: number; env?: Record<string, string | null> }
+export interface IndexInput { path: string; root?: string; source?: string; scope?: ContextScope; ttlMs?: number; maxBytes?: number; includeExtensions?: string[]; excludePaths?: string[]; followFileSymlinks?: boolean; maxDepth?: number; maxVisitedEntries?: number; maxFiles?: number; maxAggregateBytes?: number }
+export interface FetchInput { url: string; source?: string; scope?: ContextScope; ttlMs?: number; timeoutMs?: number; maxBytes?: number }
+export interface DoctorResult { ok: boolean; status: 'OK' | 'WARN' | 'FAIL'; node: string; platform: string; root: string; dbPath: string; checks: Array<{ id: string; status: 'OK' | 'WARN' | 'FAIL'; message: string; details?: unknown }> }
+
 export function contextDbPath(root?: string): string;
 export class ContextStore {
   constructor(root?: string);
   root: string;
   path: string;
   close(): void;
-  index(input: Record<string, unknown>): Record<string, unknown>;
-  search(input: Record<string, unknown>): unknown[];
-  stats(): Record<string, unknown>;
-  purge(input: Record<string, unknown>): Record<string, unknown>;
+  index(input: { id?: string; scope?: ContextScope; sessionId?: string | null; label?: string; kind?: string; text: string; metadata?: Record<string, unknown>; ttlMs?: number | null; provenance?: Record<string, unknown> }): Record<string, unknown>;
+  search(input: { query: string; scope?: ContextScope; sessionId?: string; source?: string; limit?: number }): unknown[];
+  stats(): { path: string; sources: number; chunks: number; byScope: Array<{ scope: ContextScope; count: number }> };
+  purge(input: { scope?: ContextScope; sessionId?: string; sourceId?: string }): { removed: number };
 }
-export function executeCommand(input: Record<string, any>, options?: Record<string, any>): Promise<Record<string, any>>;
-export function executeBatch(input: Record<string, any>, options?: Record<string, any>): Promise<Record<string, any>>;
-export function executeFile(input: Record<string, any>, options?: Record<string, any>): Promise<Record<string, any>>;
-export function indexFile(store: ContextStore, input: Record<string, any>, sessionId?: string): Record<string, any>;
-export function fetchAndIndex(store: ContextStore, input: Record<string, any>, sessionId?: string, signal?: AbortSignal): Promise<Record<string, any>>;
-export function runDoctor(input?: Record<string, any>): Record<string, any>;
+export function executeCommand(input: ExecutionInput, options?: { root?: string; store?: ContextStore; sessionId?: string; signal?: AbortSignal | undefined; env?: Record<string, string | undefined>; captureLimitBytes?: number }): Promise<Record<string, any>>;
+export function executeBatch(input: { commands: ExecutionInput[]; concurrency?: number; cwd?: string; timeoutMs?: number }, options?: Record<string, any>): Promise<Record<string, any>>;
+export function executeFile(input: ExecutionInput & { path: string; language: 'javascript' | 'python' | 'shell'; code: string }, options?: Record<string, any>): Promise<Record<string, any>>;
+export function indexFile(store: ContextStore, input: IndexInput, sessionId?: string, signal?: AbortSignal): Record<string, any>;
+export function fetchAndIndex(store: ContextStore, input: FetchInput, sessionId?: string, signal?: AbortSignal): Promise<Record<string, any>>;
+export function composeEnvironment(input?: { inherited?: Record<string, string | undefined>; overrides?: Record<string, string | null>; platform?: string }): { env: Record<string, string>; policy: EnvironmentPolicy };
+export const DIRECTORY_INGESTION_LIMITS: Readonly<{ maxDepth: number; maxVisitedEntries: number; maxFiles: number; maxFileBytes: number; maxAggregateBytes: number }>;
+export function walkDirectoryManifest(input: IndexInput & { signal?: AbortSignal }): DirectoryManifest;
+export function verifyManifestFile(entry: DirectoryManifestEntry, root: string): string;
+export function readManifestFile(entry: DirectoryManifestEntry, root: string, maxBytes?: number): Buffer;
+export function normalizeHtml(value: unknown, options?: { contentType?: string; maxInputBytes?: number; maxOutputBytes?: number }): HtmlNormalization;
+export function runDoctor(input?: { root?: string; dbPath?: string; stats?: Record<string, unknown>; fetchPolicy?: { mode?: string; encodings?: string[] } }): DoctorResult;
 export function projectLoad(input: Record<string, any>): Promise<Record<string, any>>;
 export function projectImpact(input: Record<string, any>): Promise<Record<string, any>>;
 export function projectInitialize(input: Record<string, any>): Promise<Record<string, any>>;

package/src/index.mjs

@@ -2,6 +2,9 @@ export { chunkContent, chunkJson, chunkMarkdown, chunkText, excerpt, normalizeTe
 export { ContextStore, contextDbPath } from './store.mjs';
 export { executeBatch, executeCommand, executeFile } from './execute.mjs';
 export { fetchAndIndex, indexFile } from './content.mjs';
+export { composeEnvironment } from './environment-policy.mjs';
+export { DIRECTORY_INGESTION_LIMITS, readManifestFile, verifyManifestFile, walkDirectoryManifest } from './directory-ingestion.mjs';
+export { normalizeHtml } from './html-normalize.mjs';
 export { runDoctor } from './doctor.mjs';
 export { createProvenanceMetadata, readProvenanceMetadata, sha256Content } from './provenance.mjs';
 export { normalizeSearchTerms, reciprocalRankFusion, rerankCandidates } from './rank.mjs';

package/src/provenance.mjs

@@ -11,7 +11,7 @@ const TRUST_BY_SOURCE_TYPE = Object.freeze({
   unknown: 'unknown',
 });
 
-const EXTRACTORS = new Set(['plain', 'markdown-v1', 'json-v1']);
+const EXTRACTORS = new Set(['plain', 'markdown-v1', 'json-v1', 'html-v1']);
 const HASH_PATTERN = /^[a-f0-9]{64}$/u;
 
 function asMetadata(value) {

package/src/server.mjs

@@ -15,7 +15,7 @@ function getStore() {
   contextStore ??= new ContextStore(root);
   return contextStore;
 }
-const server = new McpServer({ name: 'dotdotgod-context', version: '0.2.25' });
+const server = new McpServer({ name: 'dotdotgod-context', version: '0.2.26' });
 
 const scopeSchema = z.enum(['transient', 'session', 'project']).optional();
 const outputModeSchema = z.enum(['auto', 'direct', 'indexed', 'discard']).optional();
@@ -23,6 +23,7 @@ const commandSchema = {
   label: z.string().optional(), command: z.string().optional(), executable: z.string().optional(), args: z.array(z.string()).optional(),
   shell: z.boolean().optional(), cwd: z.string().optional(), timeoutMs: z.number().int().positive().optional(), outputLimit: z.number().int().positive().optional(),
   directLimit: z.number().int().positive().optional(), outputMode: outputModeSchema, scope: scopeSchema, ttlMs: z.number().int().nonnegative().optional(),
+  env: z.record(z.string(), z.string().nullable()).optional(),
 };
 
 function success(value) {
@@ -49,9 +50,11 @@ register('batch_execute', 'Run labeled local commands sequentially or with bound
 register('execute_file', 'Process a local file in an isolated child process; only bounded stdout/stderr is returned.', {
   path: z.string(), language: z.enum(['javascript', 'python', 'shell']).default('javascript'), code: z.string(), ...commandSchema,
 }, (input, extra) => executeFile(input, { root, store: getStore(), sessionId, signal: extra.signal }), { openWorldHint: false, destructiveHint: true });
-register('index', 'Index a local text file into the project-local FTS5 store without returning its raw bytes.', {
+register('index', 'Index a local text file or bounded directory into the project-local FTS5 store without returning raw bytes.', {
   path: z.string(), root: z.string().optional(), source: z.string().optional(), scope: scopeSchema, ttlMs: z.number().int().nonnegative().optional(), maxBytes: z.number().int().positive().optional(),
-}, (input) => ({ ok: true, ...indexFile(getStore(), { ...input, root }, sessionId) }), { readOnlyHint: true });
+  includeExtensions: z.array(z.string()).max(100).optional(), excludePaths: z.array(z.string()).max(500).optional(), followFileSymlinks: z.boolean().optional(),
+  maxDepth: z.number().int().nonnegative().optional(), maxVisitedEntries: z.number().int().nonnegative().optional(), maxFiles: z.number().int().nonnegative().optional(), maxAggregateBytes: z.number().int().nonnegative().optional(),
+}, (input, extra) => ({ ok: true, ...indexFile(getStore(), { ...input, root }, sessionId, extra.signal) }), { readOnlyHint: true });
 register('search', 'Search indexed command, file, and fetched content and return bounded excerpts.', {
   query: z.string().min(1), scope: scopeSchema, source: z.string().optional(), limit: z.number().int().min(1).max(50).optional(), sessionOnly: z.boolean().optional(),
 }, (input) => ({

package/src/store.mjs

@@ -1,4 +1,4 @@
-import { mkdirSync, rmSync } from 'node:fs';
+import { existsSync, mkdirSync, rmSync, statSync } from 'node:fs';
 import { dirname, join, resolve } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
 import { extname } from 'node:path';
@@ -20,40 +20,72 @@ export function contextDbPath(root) {
   return join(safeRoot(root), '.dotdotgod', 'context', 'context.sqlite');
 }
 
+function inspectSchema(db) {
+  const objects = new Map(db.prepare("SELECT name, type, sql FROM sqlite_master WHERE name IN ('sources', 'chunks')").all().map((row) => [row.name, row]));
+  const sourceRows = db.prepare('PRAGMA table_info(sources)').all();
+  const chunkRows = db.prepare('PRAGMA table_info(chunks)').all();
+  const requiredSources = ['id', 'scope', 'session_id', 'label', 'kind', 'metadata', 'created_at', 'expires_at'];
+  const requiredChunks = ['source_id', 'scope', 'session_id', 'ordinal', 'body'];
+  const missingSources = requiredSources.filter((name) => !sourceRows.some((row) => row.name === name));
+  const missingChunks = requiredChunks.filter((name) => !chunkRows.some((row) => row.name === name));
+  const sourceShape = sourceRows.length === requiredSources.length && sourceRows.every((row, index) => row.name === requiredSources[index])
+    && sourceRows[0].pk === 1
+    && sourceRows.every((row) => ['scope', 'label', 'kind', 'metadata', 'created_at'].includes(row.name) ? row.notnull === 1 : true);
+  const chunkShape = chunkRows.length === requiredChunks.length && chunkRows.every((row, index) => row.name === requiredChunks[index]);
+  const chunkSql = String(objects.get('chunks')?.sql ?? '').toLowerCase();
+  const compatible = objects.get('sources')?.type === 'table'
+    && objects.get('chunks')?.type === 'table'
+    && chunkSql.includes('using fts5')
+    && chunkSql.includes("tokenize='porter unicode61'")
+    && sourceShape
+    && chunkShape
+    && missingSources.length === 0
+    && missingChunks.length === 0;
+  return { compatible, schemaProfile: compatible ? 1 : null, missing: [...missingSources, ...missingChunks] };
+}
+
 export class ContextStore {
   constructor(root = process.cwd()) {
     this.root = safeRoot(root);
     this.path = contextDbPath(this.root);
+    const existing = existsSync(this.path);
+    if (existing) {
+      if (statSync(this.path).size === 0) throw new Error('Incompatible context database schema; empty database cannot be repaired automatically.');
+      const inspection = new DatabaseSync(this.path, { readOnly: true });
+      try {
+        const result = inspectSchema(inspection);
+        if (!result.compatible) throw new Error(`Incompatible context database schema; run doctor before repair${result.missing.length ? ` (missing: ${result.missing.join(', ')})` : ''}.`);
+      } finally {
+        inspection.close();
+      }
+    }
     mkdirSync(dirname(this.path), { recursive: true });
     this.db = new DatabaseSync(this.path);
-    this.db.exec(`
-      PRAGMA journal_mode=WAL;
-      CREATE TABLE IF NOT EXISTS sources (
-        id TEXT PRIMARY KEY,
-        scope TEXT NOT NULL,
-        session_id TEXT,
-        label TEXT NOT NULL,
-        kind TEXT NOT NULL,
-        metadata TEXT NOT NULL,
-        created_at INTEGER NOT NULL,
-        expires_at INTEGER
-      );
-      CREATE VIRTUAL TABLE IF NOT EXISTS chunks USING fts5(
-        source_id UNINDEXED,
-        scope UNINDEXED,
-        session_id UNINDEXED,
-        ordinal UNINDEXED,
-        body,
-        tokenize='porter unicode61'
-      );
-    `);
-    const sourceColumns = new Set(this.db.prepare('PRAGMA table_info(sources)').all().map((row) => row.name));
-    const chunkColumns = new Set(this.db.prepare('PRAGMA table_info(chunks)').all().map((row) => row.name));
-    const missingSources = ['id', 'scope', 'session_id', 'label', 'kind', 'metadata', 'created_at', 'expires_at'].filter((name) => !sourceColumns.has(name));
-    const missingChunks = ['source_id', 'scope', 'session_id', 'ordinal', 'body'].filter((name) => !chunkColumns.has(name));
-    if (missingSources.length || missingChunks.length) {
+    try {
+      this.db.exec('PRAGMA busy_timeout=1000; PRAGMA journal_mode=WAL;');
+      if (!existing) this.db.exec(`
+        CREATE TABLE sources (
+          id TEXT PRIMARY KEY,
+          scope TEXT NOT NULL,
+          session_id TEXT,
+          label TEXT NOT NULL,
+          kind TEXT NOT NULL,
+          metadata TEXT NOT NULL,
+          created_at INTEGER NOT NULL,
+          expires_at INTEGER
+        );
+        CREATE VIRTUAL TABLE chunks USING fts5(
+          source_id UNINDEXED,
+          scope UNINDEXED,
+          session_id UNINDEXED,
+          ordinal UNINDEXED,
+          body,
+          tokenize='porter unicode61'
+        );
+      `);
+    } catch (error) {
       this.db.close();
-      throw new Error(`Incompatible context database schema; run doctor before repair (missing: ${[...missingSources, ...missingChunks].join(', ')}).`);
+      throw error;
     }
   }
 
@@ -67,7 +99,7 @@ export class ContextStore {
     const contentType = String(provenance.contentType ?? metadata.contentType ?? '').toLowerCase();
     const extension = extname(String(metadata.path ?? '')).toLowerCase();
     const format = provenance.format ?? (contentType.includes('json') || extension === '.json' ? 'json' : contentType.includes('markdown') || ['.md', '.mdx'].includes(extension) ? 'markdown' : 'text');
-    const extractor = format === 'json' ? 'json-v1' : format === 'markdown' ? 'markdown-v1' : 'plain';
+    const extractor = provenance.extractor ?? (format === 'json' ? 'json-v1' : format === 'markdown' ? 'markdown-v1' : 'plain');
     const normalizedMetadata = createProvenanceMetadata(metadata, { sourceType, origin, content: text, extractor, indexedAt: new Date(createdAt).toISOString() });
     this.db.exec('BEGIN IMMEDIATE');
     try {
@@ -87,10 +119,18 @@ export class ContextStore {
 
   expire() {
     const ids = this.db.prepare('SELECT id FROM sources WHERE expires_at IS NOT NULL AND expires_at <= ?').all(Date.now()).map((row) => row.id);
+    if (ids.length === 0) return 0;
     const removeChunks = this.db.prepare('DELETE FROM chunks WHERE source_id = ?');
     const removeSource = this.db.prepare('DELETE FROM sources WHERE id = ?');
-    for (const id of ids) { removeChunks.run(id); removeSource.run(id); }
-    return ids.length;
+    this.db.exec('BEGIN IMMEDIATE');
+    try {
+      for (const id of ids) { removeChunks.run(id); removeSource.run(id); }
+      this.db.exec('COMMIT');
+      return ids.length;
+    } catch (error) {
+      this.db.exec('ROLLBACK');
+      throw error;
+    }
   }
 
   search({ query, scope, sessionId, source, limit = 5 }) {
@@ -168,11 +208,18 @@ export class ContextStore {
     const removeChunks = this.db.prepare('DELETE FROM chunks WHERE source_id = ?');
     const removeSource = this.db.prepare('DELETE FROM sources WHERE id = ?');
     let removed = 0;
-    for (const id of ids) {
-      removeChunks.run(id);
-      removed += Number(removeSource.run(id).changes ?? 0);
+    this.db.exec('BEGIN IMMEDIATE');
+    try {
+      for (const id of ids) {
+        removeChunks.run(id);
+        removed += Number(removeSource.run(id).changes ?? 0);
+      }
+      this.db.exec('COMMIT');
+      return { removed };
+    } catch (error) {
+      this.db.exec('ROLLBACK');
+      throw error;
     }
-    return { removed };
   }
 
   destroy() {