@bahulam/code 2.6.17 → 2.6.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bahulam/code",
-  "version": "2.6.17",
+  "version": "2.6.18",
   "description": "Bahulam Code \u2014 abundance, in your terminal. CLI-first, reliability-first, sub-agents, 65.6% SWE-bench Verified.",
   "type": "module",
   "bin": {

package/src/config/cli-args.mjs

@@ -33,7 +33,7 @@ export function parseArgs(args) {
         resume: false,
         resumeSessionId: null,
         headless: false,
-        freeswim: false,
+        skipPermissions: false,
         vision: [],
         verbose: false,
         debug: false,
@@ -106,7 +106,7 @@ export function parseArgs(args) {
 
             case '--headless':
                 result.headless = true;
-                result.freeswim = true; // headless implies skip permissions
+                result.skipPermissions = true; // headless implies skip permissions
                 break;
 
             case '--cache-report':
@@ -128,10 +128,8 @@ export function parseArgs(args) {
                 break;
             }
 
-            case '--freeswim-open-waters':
-            case '--freeswim':
-            case '--yes':
-                result.freeswim = true;
+            case '--dangerously-skip-permissions':
+                result.skipPermissions = true;
                 break;
 
             case '--verbose':
@@ -189,9 +187,7 @@ Options:
   --headless                 Non-interactive mode: auto-approve, JSONL output
   --cache-report <file>      Write prompt-cache summary JSON to <file> (headless only)
   --vision <image-path>      Attach image path in headless mode
-  --freeswim-open-waters     Skip all approval prompts (no boundaries)
-  --freeswim                 Alias for --freeswim-open-waters
-  --yes                      Alias for --freeswim-open-waters
+  --dangerously-skip-permissions  Skip ALL approval prompts, including dangerous tiers
   --verbose, -v              Verbose output
   --debug, -d                Debug mode
   --version                  Show version

package/src/context/prose-chunker.mjs

@@ -0,0 +1,255 @@
+/**
+ * Prose document chunker — JS port of the backend's Python chunker.
+ *
+ * Mirrors `codekepler-backend/app/agent/tools/shared/documents.py` so a
+ * file chunked here (CLI local path) and a file chunked server-side
+ * (chat upload or server `path` mode) produce byte-comparable chunks
+ * with the same page/chunk numbering. Callers can slice/reference
+ * chunks by (page_no, chunk_no) across surfaces.
+ *
+ * Constants come from Django's Phase 1 inline chunker
+ * (retail/chat_uploads/services.py) — the single source of truth for
+ * the whole platform.
+ *
+ * ─────────────────────────────────────────────────────────────────────
+ * Retriever interface — this module produces chunks; a retriever ranks
+ * them. Today the CLI has one retriever (BM25 via
+ * src/context/retriever.mjs). Desktop apps and richer offline setups
+ * will add an embedding-backed retriever. Both consume the same chunk
+ * shape produced here so the swap is transparent to callers:
+ *
+ *   Chunk (produced by this module):
+ *     { page: number|null, chunk_no: number, text: string, tokens: number }
+ *
+ *   Retriever (any implementation):
+ *     addSource(sourceId, chunks): void
+ *     search(query, { topK, sources? }): Array<{
+ *       sourceId, page, chunk_no, text, score
+ *     }>
+ *     removeSource(sourceId): void    // invalidate on file change
+ *
+ * The `read_attachment` tool talks to the retriever interface, never
+ * to a specific implementation. Adding embeddings later means writing
+ * an `EmbeddingRetriever` that satisfies this contract — no changes to
+ * the chunker or the tool.
+ * ─────────────────────────────────────────────────────────────────────
+ *
+ * Divergences from Python (documented):
+ *   - JS String indexing is UTF-16 code units, Python str is code points.
+ *     For BMP-only text (nearly all documents) the boundaries match
+ *     exactly. Non-BMP characters (emoji, some CJK) may fall a code unit
+ *     off vs Python; not a correctness issue for retrieval.
+ *   - TEXT_LIKE_MIMES is a JS-only superset of Python's TEXT_MIMES —
+ *     the CLI has historically supported json/yaml/html/log/rst, and
+ *     the chunker doesn't care about the surface syntax. Retrieval
+ *     quality on structured files (json/yaml) will be worse than on
+ *     prose; document that in the tool description, not here.
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+// Byte-exact match with documents.py:29-31.
+export const CHUNK_TOKENS = 800;
+export const CHUNK_OVERLAP = 100;
+export const CHARS_PER_TOKEN = 4;
+
+// Strict server-parity set — same 5 mimes as documents.py.
+export const TEXT_MIMES = new Set([
+  'text/plain',
+  'text/markdown',
+  'text/csv',
+  'text/tab-separated-values',
+]);
+export const PDF_MIMES = new Set(['application/pdf']);
+export const DOCUMENT_MIMES = new Set([...TEXT_MIMES, ...PDF_MIMES]);
+
+// Wider set the CLI already supports at the `read_attachment` tool layer.
+// Chunking works fine on any UTF-8 text — retrieval quality on structured
+// formats is up to the retriever + query.
+export const TEXT_LIKE_MIMES = new Set([
+  ...TEXT_MIMES,
+  'application/json',
+  'application/x-yaml',
+  'application/toml',
+  'text/yaml',
+  'text/html',
+  'text/x-log',
+  'text/x-rst',
+  'text/x-restructuredtext',
+]);
+
+// Extension → mime map. Kept small and explicit — Python's
+// mimetypes.guess_type varies by OS registry; this table is stable.
+const EXT_TO_MIME = new Map([
+  ['.txt', 'text/plain'],
+  ['.log', 'text/x-log'],
+  ['.md', 'text/markdown'],
+  ['.markdown', 'text/markdown'],
+  ['.mdx', 'text/markdown'],       // MDX = markdown + JSX; treat as prose for retrieval
+  ['.csv', 'text/csv'],
+  ['.tsv', 'text/tab-separated-values'],
+  ['.json', 'application/json'],
+  ['.yaml', 'application/x-yaml'],
+  ['.yml', 'application/x-yaml'],
+  ['.toml', 'application/toml'],
+  ['.html', 'text/html'],
+  ['.htm', 'text/html'],
+  ['.rst', 'text/x-rst'],
+  ['.pdf', 'application/pdf'],
+]);
+
+/**
+ * Best-effort mime for a local file (extension-only, no magic-byte sniff).
+ * Files with no matching extension return 'application/octet-stream',
+ * consistent with Python's mimetypes.
+ */
+export function guessMime(filePath) {
+  const ext = path.extname(String(filePath || '')).toLowerCase();
+  return EXT_TO_MIME.get(ext) || 'application/octet-stream';
+}
+
+/**
+ * Chunk a text string with the 800/100 sliding window (character-based).
+ * Byte-comparable to Python's chunk_text() — see documents.py:40.
+ *
+ * @param {string} text
+ * @returns {Array<{chunk_no: number, text: string, tokens: number}>}
+ */
+export function chunkText(text) {
+  if (!text) return [];
+  const stepChars = (CHUNK_TOKENS - CHUNK_OVERLAP) * CHARS_PER_TOKEN;
+  const window = CHUNK_TOKENS * CHARS_PER_TOKEN;
+  const out = [];
+  let i = 0;
+  let chunkNo = 0;
+  const len = text.length;
+  while (i < len) {
+    const segment = text.substring(i, i + window);
+    const tokens = Math.max(1, Math.floor(segment.length / CHARS_PER_TOKEN));
+    out.push({ chunk_no: chunkNo, text: segment, tokens });
+    chunkNo += 1;
+    i += stepChars;
+  }
+  return out;
+}
+
+/**
+ * Extract per-page text from PDF bytes. Returns [{page, text}] with
+ * 1-indexed page numbers. Scanned/OCR-only pages come back with empty
+ * text — callers skip those (matches Python behavior).
+ *
+ * Uses the `pdf-parse` npm dep already in the CLI. Imported from
+ * `lib/pdf-parse.js` (not the default entry) to skip the debug-hook
+ * that opens a bundled test PDF at load time and fails in production.
+ */
+export async function extractPdfPages(buffer) {
+  const { default: pdfParse } = await import('pdf-parse/lib/pdf-parse.js');
+  const pageTexts = [];
+  try {
+    await pdfParse(buffer, {
+      // pdf-parse calls this per page in page order (pageIndex is 0-based).
+      // We accumulate into an array indexed by pageIndex to be defensive
+      // against any out-of-order rendering.
+      pagerender: async (pageData) => {
+        try {
+          const content = await pageData.getTextContent();
+          const text = (content.items || [])
+            .map(item => (typeof item.str === 'string' ? item.str : ''))
+            .join(' ');
+          const idx = typeof pageData.pageIndex === 'number' ? pageData.pageIndex : pageTexts.length;
+          pageTexts[idx] = text;
+          return text;
+        } catch {
+          return '';
+        }
+      },
+    });
+  } catch {
+    return [];
+  }
+  return pageTexts.map((text, i) => ({ page: i + 1, text: text || '' }));
+}
+
+/**
+ * Extract chunks from raw bytes. Returns the same shape as Python's
+ * extract_from_bytes(): [{page, chunk_no, text, tokens}].
+ *
+ * page is null for text mimes, 1-indexed for PDF pages.
+ * chunk_no is 0-indexed and monotonic across the whole document
+ * (matches Python `global_chunk` behavior at documents.py:109-115).
+ * Unsupported mimes return an empty array.
+ *
+ * @param {Buffer} buffer
+ * @param {string} mime
+ * @param {{textMimes?: Set<string>}} [opts] override which mimes are
+ *   treated as chunkable text. Defaults to TEXT_LIKE_MIMES (CLI's
+ *   permissive set). Pass TEXT_MIMES for strict server parity.
+ */
+export async function extractFromBytes(buffer, mime, opts = {}) {
+  const normalizedMime = String(mime || '').toLowerCase();
+  const textMimes = opts.textMimes || TEXT_LIKE_MIMES;
+
+  if (textMimes.has(normalizedMime)) {
+    const text = buffer.toString('utf8');
+    return chunkText(text).map(c => ({
+      page: null,
+      chunk_no: c.chunk_no,
+      text: c.text,
+      tokens: c.tokens,
+    }));
+  }
+
+  if (PDF_MIMES.has(normalizedMime)) {
+    const pages = await extractPdfPages(buffer);
+    const out = [];
+    let globalChunk = 0;
+    for (const { page, text } of pages) {
+      if (!text || !text.trim()) continue;
+      for (const c of chunkText(text)) {
+        out.push({ page, chunk_no: globalChunk, text: c.text, tokens: c.tokens });
+        globalChunk += 1;
+      }
+    }
+    return out;
+  }
+
+  return [];
+}
+
+/**
+ * Read a local file and chunk it. Returns { mime, chunks }; both empty
+ * on unresolvable path or unsupported mime — same contract as Python's
+ * extract_from_path(). Path resolution is the caller's responsibility
+ * (the CLI uses projectRegistry.resolvePath to enforce workspace bounds
+ * before calling here).
+ *
+ * @param {string} absPath  absolute, already-resolved file path
+ * @param {{textMimes?: Set<string>}} [opts]
+ * @returns {Promise<{mime: string, chunks: Array<object>}>}
+ */
+export async function extractFromPath(absPath, opts = {}) {
+  let stat;
+  try {
+    stat = fs.statSync(absPath);
+  } catch {
+    return { mime: '', chunks: [] };
+  }
+  if (!stat.isFile()) return { mime: '', chunks: [] };
+
+  const mime = guessMime(absPath);
+  const textMimes = opts.textMimes || TEXT_LIKE_MIMES;
+  if (!textMimes.has(mime) && !PDF_MIMES.has(mime)) {
+    return { mime, chunks: [] };
+  }
+
+  let buffer;
+  try {
+    buffer = fs.readFileSync(absPath);
+  } catch {
+    return { mime, chunks: [] };
+  }
+
+  const chunks = await extractFromBytes(buffer, mime, opts);
+  return { mime, chunks };
+}

package/src/context/retriever.mjs

@@ -132,6 +132,123 @@ export class ContextRetriever {
         return true;
     }
 
+    /**
+     * Add already-chunked prose content (from prose-chunker.mjs) to the
+     * shared BM25 index — no re-read or re-chunking. Used by
+     * `read_attachment` to make docs discoverable by `search_code` /
+     * future `search_document` without a separate index.
+     *
+     * Chunk IDs are shaped `${sourceId}#c${chunk_no}` — the `#c`
+     * separator makes them distinguishable from code IDs (which use `:`)
+     * so `updateFile` for code files won't accidentally drop prose
+     * chunks and vice versa.
+     *
+     * Re-adding the same sourceId replaces its chunks (idempotent —
+     * safe to call on every `read_attachment`).
+     *
+     * @param {string} sourceId  stable id, usually a project-relative path
+     * @param {Array<{page: (number|null), chunk_no: number, text: string, tokens?: number}>} chunks
+     * @returns {number} chunks indexed
+     */
+    addProseChunks(sourceId, chunks) {
+        if (!Array.isArray(chunks) || chunks.length === 0) return 0;
+        if (!sourceId || typeof sourceId !== 'string') return 0;
+
+        if (!this.index) {
+            if (!this.loadIndex()) {
+                this.index = new BM25Index();
+                this.chunkTexts = new Map();
+            }
+        }
+
+        // Drop prior chunks for this source (idempotent).
+        const sourcePrefix = `${sourceId}#c`;
+        const oldIds = new Set();
+        for (const doc of this.index.docs) {
+            if (doc.id.startsWith(sourcePrefix)) oldIds.add(doc.id);
+        }
+        for (const id of oldIds) this.chunkTexts.delete(id);
+
+        // Collect surviving docs — reconstruct text from stored chunkTexts
+        // (BM25Index only stores tf maps + lengths, not raw text).
+        const remaining = [];
+        for (const doc of this.index.docs) {
+            if (!oldIds.has(doc.id)) {
+                remaining.push({ id: doc.id, text: this.chunkTexts.get(doc.id) || '' });
+            }
+        }
+
+        // Prep new prose docs — prefix indexed text with sourceId so BM25
+        // gets signal from the filename (mirrors how _chunkFile embeds
+        // relPath). Page tag included so page-scoped queries can hit it.
+        const newDocs = chunks.map(c => {
+            const id = `${sourceId}#c${c.chunk_no}`;
+            const pageTag = c.page != null ? ` page:${c.page}` : '';
+            const indexedText = `${sourceId}${pageTag}\n${c.text}`;
+            return { id, text: indexedText };
+        });
+
+        // Rebuild — BM25 needs IDF recomputed across all docs.
+        this.index = new BM25Index();
+        this.index.buildIndex([...remaining, ...newDocs]);
+        for (const doc of newDocs) {
+            this.chunkTexts.set(doc.id, doc.text);
+        }
+
+        // Persist. Best-effort — an unwritable indexDir shouldn't fail
+        // the tool call.
+        try {
+            if (!fs.existsSync(this.indexDir)) fs.mkdirSync(this.indexDir, { recursive: true });
+            fs.writeFileSync(path.join(this.indexDir, 'bm25.json'), JSON.stringify(this.index.toJSON()));
+            fs.writeFileSync(path.join(this.indexDir, 'chunks.json'), JSON.stringify(Object.fromEntries(this.chunkTexts)));
+        } catch { /* best-effort persist */ }
+
+        return newDocs.length;
+    }
+
+    /**
+     * Remove all chunks belonging to a source (either code path or
+     * prose sourceId). Called when a file is deleted so stale hits
+     * don't linger in the index.
+     */
+    removeSource(sourceId) {
+        if (!this.index) {
+            if (!this.loadIndex()) return 0;
+        }
+        const oldIds = new Set();
+        for (const doc of this.index.docs) {
+            // Code IDs: exact match or `sourceId:` prefix (line/AST chunks).
+            // Prose IDs: `sourceId#c` prefix.
+            if (
+                doc.id === sourceId
+                || doc.id.startsWith(`${sourceId}:`)
+                || doc.id.startsWith(`${sourceId}#c`)
+            ) {
+                oldIds.add(doc.id);
+            }
+        }
+        if (oldIds.size === 0) return 0;
+        for (const id of oldIds) this.chunkTexts.delete(id);
+
+        const remaining = [];
+        for (const doc of this.index.docs) {
+            if (!oldIds.has(doc.id)) {
+                remaining.push({ id: doc.id, text: this.chunkTexts.get(doc.id) || '' });
+            }
+        }
+
+        this.index = new BM25Index();
+        this.index.buildIndex(remaining);
+
+        try {
+            if (!fs.existsSync(this.indexDir)) fs.mkdirSync(this.indexDir, { recursive: true });
+            fs.writeFileSync(path.join(this.indexDir, 'bm25.json'), JSON.stringify(this.index.toJSON()));
+            fs.writeFileSync(path.join(this.indexDir, 'chunks.json'), JSON.stringify(Object.fromEntries(this.chunkTexts)));
+        } catch { /* best-effort */ }
+
+        return oldIds.size;
+    }
+
     /** Load persisted index. */
     loadIndex() {
         const indexPath = path.join(this.indexDir, 'bm25.json');

package/src/core/bundled-runtime.mjs

@@ -40,6 +40,7 @@ import * as fs from 'node:fs';
 import { randomBytes } from 'node:crypto';
 import * as net from 'node:net';
 import * as http from 'node:http';
+import { fileURLToPath } from 'node:url';
 
 const DEV_RUNTIME_ROOT = path.join(os.homedir(), '.bahulam', 'runtime', 'current');
 const READY_TIMEOUT_MS = 30_000;
@@ -72,7 +73,14 @@ function _runtimeRoot() {
   const arch = process.arch;       // 'arm64' | 'x64' | ...
   const siblingName = `@bahulam/runtime-${plat}-${arch}`;
   try {
-    const here = path.dirname(new URL(import.meta.url).pathname);
+    // fileURLToPath handles the Windows quirk where new URL(...).pathname
+    // returns '/C:/Users/...' with a leading slash — that broken path
+    // makes path.join produce '/C:/Users/.../node_modules/...' which
+    // fs.existsSync always returns false for, so the sibling walk fails
+    // silently and DEV_RUNTIME_ROOT is returned even when the runtime IS
+    // installed alongside. On POSIX fileURLToPath returns the plain
+    // pathname, so this is a no-op there.
+    const here = path.dirname(fileURLToPath(import.meta.url));
     // Walk up looking for node_modules containing @bahulam/runtime-<plat>-<arch>
     let dir = here;
     for (let i = 0; i < 6; i++) {

package/src/core/error-guidance.mjs

@@ -250,6 +250,22 @@ export function formatAgentErrorGuidance(data = {}) {
     lines.push(`You reached the message limit for this plan.${retry}`);
     lines.push('Upgrade your plan for a larger 5-hour message window, or switch to BYOK if available.');
     if (pricingUrl) lines.push(`Open ${pricingUrl} to upgrade.`);
+  } else if (
+    // Auth-specific gateway error → session expired / token invalid.
+    // Must match BEFORE the generic gateway branch below, otherwise
+    // 'gateway_authentication_error' falls into the retry-hint branch
+    // and the user sees "usually transient — retry" when the real fix
+    // is to re-login. The 401/token wording covers both server-shaped
+    // messages ("Invalid or expired token") and message-only fallbacks.
+    code === 'gateway_authentication_error' ||
+    code === 'authentication_error' ||
+    /401|invalid.*or.*expired.*token|invalid token|expired token|not\s*authenticated/i.test(message)
+  ) {
+    lines.push('Your session with Bahulam has expired or become invalid.');
+    lines.push('Re-authenticate:');
+    lines.push('  bahulam logout   # clears saved token');
+    lines.push('  bahulam login    # opens browser to re-auth');
+    lines.push('Then retry the command.');
   } else if (phase === 'gateway' || code.includes('gateway')) {
     // Generic first: on the platform route the user has NO provider key —
     // telling them to "check your DeepSeek API key" is wrong and alarming.
@@ -263,7 +279,7 @@ export function formatAgentErrorGuidance(data = {}) {
       lines.push('Using your own key (BYOK)? Verify it is saved and active in settings, then run /login so provider settings sync.');
     }
   } else if (/authentication|token/i.test(message)) {
-    lines.push('Run /login to re-authenticate.');
+    lines.push('Run `bahulam login` to re-authenticate.');
   } else if (/api key|openrouter/i.test(message)) {
     lines.push('Run /config to set up or refresh your provider settings.');
   } else if (/backend|network/i.test(message)) {

package/src/core/headless.mjs

@@ -147,7 +147,8 @@ export async function runHeadless({ instruction, model, timeout = 300, maxCost,
     }
     const execContext = {
         cwd: process.cwd(),
-        freeswim: true,
+        skip_permissions: true,
+        freeswim: true, // legacy wire alias — drop after cloud backend 2.7 rollout
         project_resources: projectResources,
         work_scope: buildWorkScope({
             instruction,