@poe-platform/safe-bash 0.1.818 → 0.1.820

package/dist/safe-bash-command-llm/collections.js

@@ -1,25 +1,36 @@
 import {
-  tokenInteger
-} from "../safe-bash/command-chunks/chunk-GDH7OVYK.js";
+  decodeLlmText,
+  indentLlmText,
+  withEmbeddingJsonDocument
+} from "../safe-bash/command-chunks/chunk-I7MEH76L.js";
 import {
   acquireEmbeddingInput,
   createLlmConfiguration,
-  createLlmSpool,
   embeddingCommand,
-  embeddingText,
+  embeddingText
+} from "../safe-bash/command-chunks/chunk-WOT2YXBT.js";
+import {
+  createLlmSpool,
   fileSource,
-  floatText
-} from "../safe-bash/command-chunks/chunk-7YDJ7LDO.js";
+  floatText,
+  isPythonPrintable
+} from "../safe-bash/command-chunks/chunk-BZLLQWRK.js";
+import {
+  tokenInteger
+} from "../safe-bash/command-chunks/chunk-GDH7OVYK.js";
+import "../safe-bash/command-chunks/chunk-GQV5RWMN.js";
 import {
-  jsonValue,
   sourceBytes,
   waitForSource
-} from "../safe-bash/command-chunks/chunk-UVZJAV2S.js";
+} from "../safe-bash/command-chunks/chunk-QX3WKBLI.js";
+import {
+  jsonValue
+} from "../safe-bash/command-chunks/chunk-GNPPPYDF.js";
 import "../safe-bash/command-chunks/chunk-65DS4IOB.js";
 
 // packages/safe-bash-command-llm/src/collections.ts
 import { FsError as FsError12 } from "../safe-bash-contracts/index.js";
-import { transactSqlite, withSqliteStatement as withSqliteStatement9 } from "../safe-bash-sqlite-engine/storage.js";
+import { transactSqlite, withSqliteStatement as withSqliteStatement8 } from "../safe-bash-sqlite-engine/storage.js";
 
 // packages/safe-bash-command-llm/src/collections-migrations.ts
 import { md5 } from "../safe-bash-checksum-engine/md5.js";
@@ -619,25 +630,17 @@ import { FsError as FsError4 } from "../safe-bash-contracts/index.js";
 import { pathOf } from "../safe-bash-contracts/path.js";
 
 // packages/safe-bash-command-llm/src/similar-output.ts
-function ascii(text2) {
+function ascii(text) {
   let result = "";
-  for (let index = 0; index < text2.length; index++) {
-    const code = text2.charCodeAt(index);
-    result += code > 127 ? "\\u" + code.toString(16).padStart(4, "0") : text2[index];
+  for (let index = 0; index < text.length; index++) {
+    const code = text.charCodeAt(index);
+    result += code > 127 ? "\\u" + code.toString(16).padStart(4, "0") : text[index];
   }
   return result;
 }
-async function* text(field, signal) {
-  const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
-  for await (const bytes of field.bytes) for (let offset = 0; offset < bytes.length; offset += 2048) {
-    signal.throwIfAborted();
-    yield decoder.decode(bytes.subarray(offset, offset + 2048), { stream: true });
-  }
-  yield decoder.decode();
-}
 async function* metadata(field, signal) {
   let quoted = false, escaped = false;
-  for await (const chunk of text(field, signal)) {
+  for await (const chunk of decodeLlmText(field.bytes, signal)) {
     let out = "";
     for (const char of chunk) {
       if (quoted) {
@@ -654,41 +657,6 @@ async function* metadata(field, signal) {
     yield out;
   }
 }
-async function* indent(field, signal) {
-  const replay = text(field, signal)[Symbol.asyncIterator]();
-  let pending = "", offset = 0;
-  async function* line(length2, nonblank2) {
-    if (nonblank2) yield "  ";
-    while (length2) {
-      if (offset === pending.length) {
-        const next = await replay.next();
-        if (next.done) throw new Error("Stored content changed");
-        pending = next.value;
-        offset = 0;
-        if (!pending.length) continue;
-      }
-      const count = Math.min(length2, pending.length - offset);
-      yield pending.slice(offset, offset + count);
-      offset += count;
-      length2 -= count;
-    }
-  }
-  let length = 0, nonblank = false;
-  try {
-    for await (const chunk of text(field, signal)) for (const char of chunk) {
-      length += char.length;
-      if (char === "\uFEFF" || char.trim() && !["", "", "", "", "\x85"].includes(char)) nonblank = true;
-      if (["\n", "\r", "\v", "\f", "", "", "", "\x85", "\u2028", "\u2029"].includes(char)) {
-        yield* line(length, nonblank);
-        length = 0;
-        nonblank = false;
-      }
-    }
-    if (length) yield* line(length, nonblank);
-  } finally {
-    await replay.return?.();
-  }
-}
 async function* similarOutput(row, plain, signal) {
   const encoder = new TextEncoder();
   async function* chunks() {
@@ -698,7 +666,7 @@ async function* similarOutput(row, plain, signal) {
 
 `;
       if (row.content?.size) {
-        yield* indent(row.content, signal);
+        yield* indentLlmText(row.content.bytes, signal);
         yield "\n";
       }
       if (row.metadata?.size) {
@@ -712,7 +680,7 @@ async function* similarOutput(row, plain, signal) {
     yield `{"id": ${ascii(JSON.stringify(row.id))}, "score": ${score}, "content": `;
     if (row.content) {
       yield '"';
-      for await (const chunk of text(row.content, signal)) yield ascii(JSON.stringify(chunk).slice(1, -1));
+      for await (const chunk of decodeLlmText(row.content.bytes, signal)) yield ascii(JSON.stringify(chunk).slice(1, -1));
       yield '"';
     } else yield "null";
     yield ', "metadata": ';
@@ -925,7 +893,7 @@ async function withCsvEmbeddingEntries(options, input, operation) {
       await session.execute("CREATE TABLE headers(name TEXT PRIMARY KEY,first INTEGER UNIQUE,last INTEGER); CREATE TABLE fields(position INTEGER PRIMARY KEY,start INTEGER,end INTEGER)");
       const budget = new CsvBudget({}, signal), encoder = new TextEncoder();
       let events = [];
-      const parser = new CsvParser(detected?.dialect ?? { tabs: options.tabs ?? false }, budget, { text: (text2) => events.push({ text: text2 }), field: () => events.push({ field: true }), row: () => events.push({ row: true }) });
+      const parser = new CsvParser(detected?.dialect ?? { tabs: options.tabs ?? false }, budget, { text: (text) => events.push({ text }), field: () => events.push({ field: true }), row: () => events.push({ row: true }) });
       let header = true, name = "", column = 0, columns = 0, start = 0, size = 0;
       let spool;
       const query = async (sql, bindings) => withSqliteStatement6(session.module, { ...session, signal, sql }, async (statement) => {
@@ -1062,119 +1030,6 @@ async function withCsvEmbeddingEntries(options, input, operation) {
 
 // packages/safe-bash-command-llm/src/import-json.ts
 import { FsError as FsError7 } from "../safe-bash-contracts/index.js";
-
-// packages/safe-bash-command-llm/src/import-json-document.ts
-import { jsonValues, decodeUtf8, encodePythonCodePoints } from "../safe-bash-query-engine/input.js";
-import { decodeLatin1 } from "../safe-bash-query-engine/bytes.js";
-import { Budget, resolveJqLimits } from "../safe-bash-query-engine/limits.js";
-import { Decimal } from "../safe-bash-query-engine/numbers.js";
-import { createPrivateSqliteStorage as createPrivateSqliteStorage2, withPrivateSqliteSession as withPrivateSqliteSession2, withSqliteStatement as withSqliteStatement7 } from "../safe-bash-sqlite-engine/storage.js";
-async function withEmbeddingJsonDocument(options, input, operation) {
-  const { fs, directory, signal } = options;
-  const storage = await createPrivateSqliteStorage2({ ...options, maxFiles: options.maxOpenFiles });
-  try {
-    return await withPrivateSqliteSession2({ ...options, fs: storage.fs, directory: storage.directory, path: storage.directory + "/json" }, async (session) => {
-      await session.execute("CREATE TABLE nodes(id INTEGER PRIMARY KEY,type TEXT,start INTEGER,end INTEGER,token TEXT); CREATE TABLE children(parent INTEGER,key BLOB,node INTEGER,position INTEGER,PRIMARY KEY(parent,key)); CREATE INDEX child_order ON children(parent,position)");
-      const spool = await createLlmSpool(fs, directory, signal, "input");
-      try {
-        const execute = async (sql, bindings) => withSqliteStatement7(session.module, { ...session, signal, sql }, async (statement) => {
-          for await (const ignored of statement.rows(bindings, [])) signal.throwIfAborted();
-        });
-        let serial = 0, size = 0, root, stringNode;
-        const stack = [];
-        const insert = async (type, key, token = "", keyPoints) => {
-          const node = { id: serial++, type, start: size, end: size, token };
-          await execute("INSERT INTO nodes VALUES(?,?,?,?,?)", [node.id, type, size, size, token]);
-          const parent = stack.at(-1);
-          if (parent === void 0) root = node;
-          else await execute("INSERT INTO children VALUES(?,?,?,?) ON CONFLICT(parent,key) DO UPDATE SET node=excluded.node", [parent, keyPoints ? encodePythonCodePoints([1, ...keyPoints]) : new TextEncoder().encode("0" + String(key)), node.id, node.id]);
-          return node;
-        };
-        const budget = new Budget(resolveJqLimits({ maxInputBytes: Infinity, maxValueBytes: Infinity, maxCollectionSize: Infinity, maxSteps: Infinity }), signal);
-        for await (const event of jsonValues(input, budget, { stream: true, stringChunks: { maxControlBytes: 65536, containers: true, codePoints: true }, profile: "python39" })) {
-          signal.throwIfAborted();
-          if (!Array.isArray(event) || !Array.isArray(event[0])) throw new Error("Invalid JSON parser event");
-          const path = event[0], value = event[1], kind = event[2], metadata2 = event[3];
-          if (kind === "open") {
-            const node = await insert(value === "{" ? "object" : "array", path.at(-1), "", metadata2.key);
-            stack.push(node.id);
-          } else if (kind === "close") stack.pop();
-          else if (typeof kind === "boolean") {
-            if (typeof value !== "string") throw new Error("Invalid JSON string event");
-            stringNode ??= await insert("string", path.at(-1), "", metadata2.key);
-            const points2 = metadata2.points;
-            if (!points2) throw new Error("Missing JSON code points");
-            const bytes = new Uint8Array(points2.length * 4), view = new DataView(bytes.buffer);
-            for (let i = 0; i < points2.length; i++) view.setUint32(i * 4, points2[i], true);
-            await spool.write(bytes);
-            size += bytes.length;
-            if (!Number.isSafeInteger(size)) throw new RangeError("JSON staging byte range exceeds safe integer");
-            if (kind) {
-              await execute("UPDATE nodes SET end=? WHERE id=?", [size, stringNode.id]);
-              if (stringNode.id === root?.id) root = { ...stringNode, end: size };
-              stringNode = void 0;
-            }
-          } else if (kind === null) {
-            if (value instanceof Decimal) await insert("number", path.at(-1), value.text, metadata2.key);
-            else if (typeof value === "number") await insert("number", path.at(-1), String(value), metadata2.key);
-            else if (value === null) await insert("null", path.at(-1), "", metadata2.key);
-            else if (typeof value === "boolean") await insert("boolean", path.at(-1), String(value), metadata2.key);
-          }
-        }
-        async function* points(node) {
-          if (node.type !== "string") throw new TypeError("JSON node is not a string");
-          let unit = 0, count = 0;
-          for await (const bytes of spool.replay(async () => ({ start: node.start, end: node.end }))) {
-            const values = [];
-            for (const byte of bytes) {
-              unit += byte * 2 ** (count++ * 8);
-              if (count === 4) {
-                values.push(unit);
-                unit = 0;
-                count = 0;
-              }
-            }
-            if (values.length) yield values;
-          }
-          if (count) throw new Error("Truncated JSON staging code point");
-        }
-        if (!root) throw new Error("JSON document is empty");
-        return await operation({ root, async child(parent, after = -1) {
-          let child;
-          await withSqliteStatement7(session.module, { ...session, signal, sql: "SELECT n.id,n.type,n.start,n.end,n.token,c.position,c.key FROM children c JOIN nodes n ON n.id=c.node WHERE c.parent=? AND c.position>? ORDER BY c.position LIMIT 1" }, async (statement) => {
-            for await (const [id, type, start, end, token, position, key] of statement.rows([parent, after], ["integer", "text", "integer", "integer", "text", "integer", "blob"])) {
-              const bytes = key, keyPoints = [];
-              const decoded = decodeUtf8(decodeLatin1(bytes), budget, "surrogatepass", keyPoints);
-              child = { node: { id: Number(id), type, start: Number(start), end: Number(end), token: String(token) }, position: Number(position), key: keyPoints[0] === 1 ? decoded.slice(1) : Number(decoded.slice(1)), ...keyPoints[0] === 1 ? { keyPoints: keyPoints.slice(1) } : {} };
-            }
-          });
-          return child;
-        }, points, async *text(node) {
-          for await (const values of points(node)) yield String.fromCodePoint(...values);
-        } });
-      } finally {
-        await spool.close();
-      }
-    });
-  } finally {
-    await storage.close();
-  }
-}
-
-// packages/safe-bash-command-llm/src/python-printable.ts
-var excluded = [0, 31, 127, 160, 173, 173, 888, 889, 896, 899, 907, 907, 909, 909, 930, 930, 1328, 1328, 1367, 1368, 1419, 1420, 1424, 1424, 1480, 1487, 1515, 1518, 1525, 1541, 1564, 1565, 1757, 1757, 1806, 1807, 1867, 1868, 1970, 1983, 2043, 2044, 2094, 2095, 2111, 2111, 2140, 2141, 2143, 2143, 2155, 2207, 2229, 2229, 2248, 2258, 2274, 2274, 2436, 2436, 2445, 2446, 2449, 2450, 2473, 2473, 2481, 2481, 2483, 2485, 2490, 2491, 2501, 2502, 2505, 2506, 2511, 2518, 2520, 2523, 2526, 2526, 2532, 2533, 2559, 2560, 2564, 2564, 2571, 2574, 2577, 2578, 2601, 2601, 2609, 2609, 2612, 2612, 2615, 2615, 2618, 2619, 2621, 2621, 2627, 2630, 2633, 2634, 2638, 2640, 2642, 2648, 2653, 2653, 2655, 2661, 2679, 2688, 2692, 2692, 2702, 2702, 2706, 2706, 2729, 2729, 2737, 2737, 2740, 2740, 2746, 2747, 2758, 2758, 2762, 2762, 2766, 2767, 2769, 2783, 2788, 2789, 2802, 2808, 2816, 2816, 2820, 2820, 2829, 2830, 2833, 2834, 2857, 2857, 2865, 2865, 2868, 2868, 2874, 2875, 2885, 2886, 2889, 2890, 2894, 2900, 2904, 2907, 2910, 2910, 2916, 2917, 2936, 2945, 2948, 2948, 2955, 2957, 2961, 2961, 2966, 2968, 2971, 2971, 2973, 2973, 2976, 2978, 2981, 2983, 2987, 2989, 3002, 3005, 3011, 3013, 3017, 3017, 3022, 3023, 3025, 3030, 3032, 3045, 3067, 3071, 3085, 3085, 3089, 3089, 3113, 3113, 3130, 3132, 3141, 3141, 3145, 3145, 3150, 3156, 3159, 3159, 3163, 3167, 3172, 3173, 3184, 3190, 3213, 3213, 3217, 3217, 3241, 3241, 3252, 3252, 3258, 3259, 3269, 3269, 3273, 3273, 3278, 3284, 3287, 3293, 3295, 3295, 3300, 3301, 3312, 3312, 3315, 3327, 3341, 3341, 3345, 3345, 3397, 3397, 3401, 3401, 3408, 3411, 3428, 3429, 3456, 3456, 3460, 3460, 3479, 3481, 3506, 3506, 3516, 3516, 3518, 3519, 3527, 3529, 3531, 3534, 3541, 3541, 3543, 3543, 3552, 3557, 3568, 3569, 3573, 3584, 3643, 3646, 3676, 3712, 3715, 3715, 3717, 3717, 3723, 3723, 3748, 3748, 3750, 3750, 3774, 3775, 3781, 3781, 3783, 3783, 3790, 3791, 3802, 3803, 3808, 3839, 3912, 3912, 3949, 3952, 3992, 3992, 4029, 4029, 4045, 4045, 4059, 4095, 4294, 4294, 4296, 4300, 4302, 4303, 4681, 4681, 4686, 4687, 4695, 4695, 4697, 4697, 4702, 4703, 4745, 4745, 4750, 4751, 4785, 4785, 4790, 4791, 4799, 4799, 4801, 4801, 4806, 4807, 4823, 4823, 4881, 4881, 4886, 4887, 4955, 4956, 4989, 4991, 5018, 5023, 5110, 5111, 5118, 5119, 5760, 5760, 5789, 5791, 5881, 5887, 5901, 5901, 5909, 5919, 5943, 5951, 5972, 5983, 5997, 5997, 6001, 6001, 6004, 6015, 6110, 6111, 6122, 6127, 6138, 6143, 6158, 6159, 6170, 6175, 6265, 6271, 6315, 6319, 6390, 6399, 6431, 6431, 6444, 6447, 6460, 6463, 6465, 6467, 6510, 6511, 6517, 6527, 6572, 6575, 6602, 6607, 6619, 6621, 6684, 6685, 6751, 6751, 6781, 6782, 6794, 6799, 6810, 6815, 6830, 6831, 6849, 6911, 6988, 6991, 7037, 7039, 7156, 7163, 7224, 7226, 7242, 7244, 7305, 7311, 7355, 7356, 7368, 7375, 7419, 7423, 7674, 7674, 7958, 7959, 7966, 7967, 8006, 8007, 8014, 8015, 8024, 8024, 8026, 8026, 8028, 8028, 8030, 8030, 8062, 8063, 8117, 8117, 8133, 8133, 8148, 8149, 8156, 8156, 8176, 8177, 8181, 8181, 8191, 8207, 8232, 8239, 8287, 8303, 8306, 8307, 8335, 8335, 8349, 8351, 8384, 8399, 8433, 8447, 8588, 8591, 9255, 9279, 9291, 9311, 11124, 11125, 11158, 11158, 11311, 11311, 11359, 11359, 11508, 11512, 11558, 11558, 11560, 11564, 11566, 11567, 11624, 11630, 11633, 11646, 11671, 11679, 11687, 11687, 11695, 11695, 11703, 11703, 11711, 11711, 11719, 11719, 11727, 11727, 11735, 11735, 11743, 11743, 11859, 11903, 11930, 11930, 12020, 12031, 12246, 12271, 12284, 12288, 12352, 12352, 12439, 12440, 12544, 12548, 12592, 12592, 12687, 12687, 12772, 12783, 12831, 12831, 40957, 40959, 42125, 42127, 42183, 42191, 42540, 42559, 42744, 42751, 42944, 42945, 42955, 42996, 43053, 43055, 43066, 43071, 43128, 43135, 43206, 43213, 43226, 43231, 43348, 43358, 43389, 43391, 43470, 43470, 43482, 43485, 43519, 43519, 43575, 43583, 43598, 43599, 43610, 43611, 43715, 43738, 43767, 43776, 43783, 43784, 43791, 43792, 43799, 43807, 43815, 43815, 43823, 43823, 43884, 43887, 44014, 44015, 44026, 44031, 55204, 55215, 55239, 55242, 55292, 63743, 64110, 64111, 64218, 64255, 64263, 64274, 64280, 64284, 64311, 64311, 64317, 64317, 64319, 64319, 64322, 64322, 64325, 64325, 64450, 64466, 64832, 64847, 64912, 64913, 64968, 65007, 65022, 65023, 65050, 65055, 65107, 65107, 65127, 65127, 65132, 65135, 65141, 65141, 65277, 65280, 65471, 65473, 65480, 65481, 65488, 65489, 65496, 65497, 65501, 65503, 65511, 65511, 65519, 65531, 65534, 65535, 65548, 65548, 65575, 65575, 65595, 65595, 65598, 65598, 65614, 65615, 65630, 65663, 65787, 65791, 65795, 65798, 65844, 65846, 65935, 65935, 65949, 65951, 65953, 65999, 66046, 66175, 66205, 66207, 66257, 66271, 66300, 66303, 66340, 66348, 66379, 66383, 66427, 66431, 66462, 66462, 66500, 66503, 66518, 66559, 66718, 66719, 66730, 66735, 66772, 66775, 66812, 66815, 66856, 66863, 66916, 66926, 66928, 67071, 67383, 67391, 67414, 67423, 67432, 67583, 67590, 67591, 67593, 67593, 67638, 67638, 67641, 67643, 67645, 67646, 67670, 67670, 67743, 67750, 67760, 67807, 67827, 67827, 67830, 67834, 67868, 67870, 67898, 67902, 67904, 67967, 68024, 68027, 68048, 68049, 68100, 68100, 68103, 68107, 68116, 68116, 68120, 68120, 68150, 68151, 68155, 68158, 68169, 68175, 68185, 68191, 68256, 68287, 68327, 68330, 68343, 68351, 68406, 68408, 68438, 68439, 68467, 68471, 68498, 68504, 68509, 68520, 68528, 68607, 68681, 68735, 68787, 68799, 68851, 68857, 68904, 68911, 68922, 69215, 69247, 69247, 69290, 69290, 69294, 69295, 69298, 69375, 69416, 69423, 69466, 69551, 69580, 69599, 69623, 69631, 69710, 69713, 69744, 69758, 69821, 69821, 69826, 69839, 69865, 69871, 69882, 69887, 69941, 69941, 69960, 69967, 70007, 70015, 70112, 70112, 70133, 70143, 70162, 70162, 70207, 70271, 70279, 70279, 70281, 70281, 70286, 70286, 70302, 70302, 70314, 70319, 70379, 70383, 70394, 70399, 70404, 70404, 70413, 70414, 70417, 70418, 70441, 70441, 70449, 70449, 70452, 70452, 70458, 70458, 70469, 70470, 70473, 70474, 70478, 70479, 70481, 70486, 70488, 70492, 70500, 70501, 70509, 70511, 70517, 70655, 70748, 70748, 70754, 70783, 70856, 70863, 70874, 71039, 71094, 71095, 71134, 71167, 71237, 71247, 71258, 71263, 71277, 71295, 71353, 71359, 71370, 71423, 71451, 71452, 71468, 71471, 71488, 71679, 71740, 71839, 71923, 71934, 71943, 71944, 71946, 71947, 71956, 71956, 71959, 71959, 71990, 71990, 71993, 71994, 72007, 72015, 72026, 72095, 72104, 72105, 72152, 72153, 72165, 72191, 72264, 72271, 72355, 72383, 72441, 72703, 72713, 72713, 72759, 72759, 72774, 72783, 72813, 72815, 72848, 72849, 72872, 72872, 72887, 72959, 72967, 72967, 72970, 72970, 73015, 73017, 73019, 73019, 73022, 73022, 73032, 73039, 73050, 73055, 73062, 73062, 73065, 73065, 73103, 73103, 73106, 73106, 73113, 73119, 73130, 73439, 73465, 73647, 73649, 73663, 73714, 73726, 74650, 74751, 74863, 74863, 74869, 74879, 75076, 77823, 78895, 82943, 83527, 92159, 92729, 92735, 92767, 92767, 92778, 92781, 92784, 92879, 92910, 92911, 92918, 92927, 92998, 93007, 93018, 93018, 93026, 93026, 93048, 93052, 93072, 93759, 93851, 93951, 94027, 94030, 94088, 94094, 94112, 94175, 94181, 94191, 94194, 94207, 100344, 100351, 101590, 101631, 101641, 110591, 110879, 110927, 110931, 110947, 110952, 110959, 111356, 113663, 113771, 113775, 113789, 113791, 113801, 113807, 113818, 113819, 113824, 118783, 119030, 119039, 119079, 119080, 119155, 119162, 119273, 119295, 119366, 119519, 119540, 119551, 119639, 119647, 119673, 119807, 119893, 119893, 119965, 119965, 119968, 119969, 119971, 119972, 119975, 119976, 119981, 119981, 119994, 119994, 119996, 119996, 120004, 120004, 120070, 120070, 120075, 120076, 120085, 120085, 120093, 120093, 120122, 120122, 120127, 120127, 120133, 120133, 120135, 120137, 120145, 120145, 120486, 120487, 120780, 120781, 121484, 121498, 121504, 121504, 121520, 122879, 122887, 122887, 122905, 122906, 122914, 122914, 122917, 122917, 122923, 123135, 123181, 123183, 123198, 123199, 123210, 123213, 123216, 123583, 123642, 123646, 123648, 124927, 125125, 125126, 125143, 125183, 125260, 125263, 125274, 125277, 125280, 126064, 126133, 126208, 126270, 126463, 126468, 126468, 126496, 126496, 126499, 126499, 126501, 126502, 126504, 126504, 126515, 126515, 126520, 126520, 126522, 126522, 126524, 126529, 126531, 126534, 126536, 126536, 126538, 126538, 126540, 126540, 126544, 126544, 126547, 126547, 126549, 126550, 126552, 126552, 126554, 126554, 126556, 126556, 126558, 126558, 126560, 126560, 126563, 126563, 126565, 126566, 126571, 126571, 126579, 126579, 126584, 126584, 126589, 126589, 126591, 126591, 126602, 126602, 126620, 126624, 126628, 126628, 126634, 126634, 126652, 126703, 126706, 126975, 127020, 127023, 127124, 127135, 127151, 127152, 127168, 127168, 127184, 127184, 127222, 127231, 127406, 127461, 127491, 127503, 127548, 127551, 127561, 127567, 127570, 127583, 127590, 127743, 128728, 128735, 128749, 128751, 128765, 128767, 128884, 128895, 128985, 128991, 129004, 129023, 129036, 129039, 129096, 129103, 129114, 129119, 129160, 129167, 129198, 129199, 129202, 129279, 129401, 129401, 129484, 129484, 129620, 129631, 129646, 129647, 129653, 129655, 129659, 129663, 129671, 129679, 129705, 129711, 129719, 129727, 129731, 129743, 129751, 129791, 129939, 129939, 129995, 130031, 130042, 131071, 173790, 173823, 177973, 177983, 178206, 178207, 183970, 183983, 191457, 194559, 195102, 196607, 201547, 917759, 918e3, 1114111];
-function isPythonPrintable(point) {
-  let low = 0, high = excluded.length / 2;
-  while (low < high) {
-    const middle = low + high >>> 1;
-    if (point < excluded[middle * 2]) high = middle;
-    else if (point > excluded[middle * 2 + 1]) low = middle + 1;
-    else return false;
-  }
-  return true;
-}
-
-// packages/safe-bash-command-llm/src/import-json.ts
 async function withJsonEmbeddingEntries(options, input, operation) {
   return withEmbeddingJsonDocument(options, input, async (document) => {
     if (document.root.type !== "object" && document.root.type !== "array") throw new TypeError("JSON must be a list or a dictionary");
@@ -1265,9 +1120,10 @@ async function withEmbeddingJsonRows(options, document, operation, singleRow = f
   const entries = { async *[Symbol.asyncIterator]() {
     let position = -1;
     while (true) {
-      const row = singleRow || document.root.type === "object" ? position < 0 ? document.root : void 0 : (await document.child(document.root.id, position))?.node;
+      const child = singleRow || document.root.type === "object" ? void 0 : await document.child(document.root.id, position);
+      const row = singleRow || document.root.type === "object" ? position < 0 ? document.root : void 0 : child?.node;
       if (!row) break;
-      position = row.id;
+      position = child?.position ?? row.id;
       if (row.type !== "object") throw new TypeError(`'${row.type === "array" ? "list" : row.type === "null" ? "NoneType" : row.type === "boolean" ? "bool" : row.type === "string" ? "str" : floating(row.token) ? "float" : "int"}' object has no attribute 'values'`);
       const first = await document.child(row.id);
       if (!first) throw new RangeError("list index out of range");
@@ -1319,7 +1175,7 @@ async function withEmbeddingJsonRows(options, document, operation, singleRow = f
 // packages/safe-bash-command-llm/src/import-glob.ts
 import { FsError as FsError8 } from "../safe-bash-contracts/index.js";
 import { compilePythonGlob } from "../safe-bash-regex-engine/python-glob.js";
-import { createPrivateSqliteStorage as createPrivateSqliteStorage3, withPrivateSqliteSession as withPrivateSqliteSession3, withSqliteStatement as withSqliteStatement8 } from "../safe-bash-sqlite-engine/storage.js";
+import { createPrivateSqliteStorage as createPrivateSqliteStorage2, withPrivateSqliteSession as withPrivateSqliteSession2, withSqliteStatement as withSqliteStatement7 } from "../safe-bash-sqlite-engine/storage.js";
 async function withEmbeddingFileGlob(options, source, operation) {
   const { fs, signal } = options;
   signal.throwIfAborted();
@@ -1337,12 +1193,12 @@ async function withEmbeddingFileGlob(options, source, operation) {
   const sources = single ? [single] : source;
   if (!fs.iterateDirectory) throw new FsError8("ENOTSUP", { message: "File glob requires streaming directory enumeration" });
   const ignored = (error) => error instanceof FsError8 && ["ENOENT", "ENOTDIR", "EACCES"].includes(error.code);
-  const storage = await createPrivateSqliteStorage3({ ...options, maxFiles: options.maxOpenFiles });
+  const storage = await createPrivateSqliteStorage2({ ...options, maxFiles: options.maxOpenFiles });
   let failed = false;
   try {
-    return await withPrivateSqliteSession3({ ...options, fs: storage.fs, directory: storage.directory, path: storage.directory + "/glob" }, async (session) => {
+    return await withPrivateSqliteSession2({ ...options, fs: storage.fs, directory: storage.directory, path: storage.directory + "/glob" }, async (session) => {
       await session.execute("CREATE TABLE stack(seq INTEGER PRIMARY KEY AUTOINCREMENT,path TEXT,id TEXT,part INTEGER); CREATE TABLE children(seq INTEGER PRIMARY KEY,path TEXT,id TEXT,part INTEGER); CREATE TABLE visited(id TEXT,part INTEGER,PRIMARY KEY(id,part)); CREATE TABLE emitted(id TEXT PRIMARY KEY)");
-      const query = async (sql, bindings = [], columns = []) => withSqliteStatement8(session.module, { ...session, signal, sql }, async (statement) => {
+      const query = async (sql, bindings = [], columns = []) => withSqliteStatement7(session.module, { ...session, signal, sql }, async (statement) => {
         let result;
         for await (const row of statement.rows(bindings, columns)) {
           if (result) throw new Error("Glob control query returned more than one row");
@@ -1869,13 +1725,13 @@ function createLlmCollectionCommands(options) {
     if (group === "similar") return similarCommand(invocation, { ...limits, now });
     if (group === "embed-multi") return embedMultiCommand(invocation, { ...limits, now });
     const encoder = new TextEncoder();
-    const emit = async (text2) => {
-      for (let offset = 0; offset < text2.length; ) {
+    const emit = async (text) => {
+      for (let offset = 0; offset < text.length; ) {
         await step();
-        let end = Math.min(offset + 4096, text2.length);
-        const last = text2.charCodeAt(end - 1);
-        if (end < text2.length && last >= 55296 && last <= 56319) end--;
-        await write(encoder.encode(text2.slice(offset, end)));
+        let end = Math.min(offset + 4096, text.length);
+        const last = text.charCodeAt(end - 1);
+        if (end < text.length && last >= 55296 && last <= 56319) end--;
+        await write(encoder.encode(text.slice(offset, end)));
         offset = end;
       }
     };
@@ -2036,7 +1892,7 @@ var schema = [
 ];
 async function initialize(session, signal, now, create) {
   const tables = [];
-  await withSqliteStatement9(session.module, { ...session, signal, sql: "SELECT name FROM sqlite_schema WHERE type='table' AND name IN ('collections','embeddings')" }, async (query) => {
+  await withSqliteStatement8(session.module, { ...session, signal, sql: "SELECT name FROM sqlite_schema WHERE type='table' AND name IN ('collections','embeddings')" }, async (query) => {
     for await (const [name] of query.rows([], ["text"])) tables.push(name);
   });
   if (!tables.length) {
@@ -2044,12 +1900,12 @@ async function initialize(session, signal, now, create) {
     await session.execute(schema[0].replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS"));
     for (const statement of schema.slice(1)) await session.execute(statement);
     const timestamp = now().toISOString().replace("T", " ").replace("Z", "000+00:00");
-    await withSqliteStatement9(session.module, { ...session, signal, sql: "INSERT INTO _sqlite_migrations(migration_set,name,applied_at) VALUES (?,?,?)" }, async (query) => {
+    await withSqliteStatement8(session.module, { ...session, signal, sql: "INSERT INTO _sqlite_migrations(migration_set,name,applied_at) VALUES (?,?,?)" }, async (query) => {
       for (const name of migrations) for await (const row of query.rows(["llm.embeddings", name, timestamp], [])) void row;
     });
   } else {
     const columns = [];
-    await withSqliteStatement9(session.module, { ...session, signal, sql: "SELECT name FROM pragma_table_info('embeddings') ORDER BY cid" }, async (query) => {
+    await withSqliteStatement8(session.module, { ...session, signal, sql: "SELECT name FROM pragma_table_info('embeddings') ORDER BY cid" }, async (query) => {
       for await (const [name] of query.rows([], ["text"])) columns.push(name);
     });
     if (tables.length !== 2) throw new Error("Incomplete embedding database schema");
@@ -2096,7 +1952,7 @@ async function withLlmCollections(options, operation) {
     });
     const lookup = async (session, name) => {
       if (typeof name !== "string") throw new TypeError("Collection name must be a string");
-      return withSqliteStatement9(session.module, { ...session, signal, sql: "SELECT id,name,model FROM collections WHERE name=?" }, async (query) => {
+      return withSqliteStatement8(session.module, { ...session, signal, sql: "SELECT id,name,model FROM collections WHERE name=?" }, async (query) => {
         for await (const [id, storedName, model] of query.rows([name], ["integer", "text", "text"])) return { id, name: storedName, model };
         return void 0;
       });
@@ -2112,7 +1968,7 @@ async function withLlmCollections(options, operation) {
           if (settings.create === false) throw new LlmCollectionDoesNotExist(name);
           if (settings.model === void 0) throw new Error("Either model= or model_id= must be provided when creating a new collection");
           if (typeof settings.model !== "string") throw new TypeError("Collection model must be a string");
-          return withSqliteStatement9(session.module, { ...session, signal, sql: "INSERT INTO collections(name,model) VALUES (?,?) RETURNING id" }, async (query) => {
+          return withSqliteStatement8(session.module, { ...session, signal, sql: "INSERT INTO collections(name,model) VALUES (?,?) RETURNING id" }, async (query) => {
             for await (const [id] of query.rows([name, settings.model], ["integer"])) return { id, name, model: settings.model };
             throw new Error("Collection insertion returned no ID");
           });
@@ -2120,7 +1976,7 @@ async function withLlmCollections(options, operation) {
       },
       list(visit) {
         return native(async (session) => {
-          await withSqliteStatement9(session.module, { ...session, signal, sql: "SELECT id,name,model,(SELECT count(id) FROM embeddings WHERE collection_id=collections.id) FROM collections ORDER BY name,model" }, async (query) => {
+          await withSqliteStatement8(session.module, { ...session, signal, sql: "SELECT id,name,model,(SELECT count(id) FROM embeddings WHERE collection_id=collections.id) FROM collections ORDER BY name,model" }, async (query) => {
             for await (const [id, name, model, count] of query.rows([], ["integer", "text", "text", "integer"])) await visit({ id, name, model, count });
           });
         });
@@ -2129,7 +1985,7 @@ async function withLlmCollections(options, operation) {
         return native(async (session) => {
           const existing = await lookup(session, name);
           if (!existing) throw new LlmCollectionDoesNotExist(name);
-          for (const sql of ["DELETE FROM embeddings WHERE collection_id=?", "DELETE FROM collections WHERE id=?"]) await withSqliteStatement9(session.module, { ...session, signal, sql }, async (query) => {
+          for (const sql of ["DELETE FROM embeddings WHERE collection_id=?", "DELETE FROM collections WHERE id=?"]) await withSqliteStatement8(session.module, { ...session, signal, sql }, async (query) => {
             for await (const row of query.rows([existing.id], [])) void row;
           });
         });

package/dist/safe-bash-command-llm/command.js

@@ -3,12 +3,20 @@ import {
   createLlmCommands,
   llmCommands,
   llmReferenceVersion
-} from "../safe-bash/command-chunks/chunk-KSM5CCHT.js";
-import "../safe-bash/command-chunks/chunk-7ITPO2V6.js";
-import "../safe-bash/command-chunks/chunk-7YDJ7LDO.js";
-import "../safe-bash/command-chunks/chunk-2ZWRW6UQ.js";
-import "../safe-bash/command-chunks/chunk-OTEHQQU7.js";
-import "../safe-bash/command-chunks/chunk-UVZJAV2S.js";
+} from "../safe-bash/command-chunks/chunk-FVHPDYV5.js";
+import "../safe-bash/command-chunks/chunk-MCUDSE4H.js";
+import "../safe-bash/command-chunks/chunk-WOT2YXBT.js";
+import "../safe-bash/command-chunks/chunk-BZLLQWRK.js";
+import "../safe-bash/command-chunks/chunk-YNFQ5AXM.js";
+import "../safe-bash/command-chunks/chunk-3PO534WJ.js";
+import "../safe-bash/command-chunks/chunk-B74LVFAN.js";
+import "../safe-bash/command-chunks/chunk-GDH7OVYK.js";
+import "../safe-bash/command-chunks/chunk-GQV5RWMN.js";
+import "../safe-bash/command-chunks/chunk-CKQ5SXOO.js";
+import "../safe-bash/command-chunks/chunk-B6NAEWHC.js";
+import "../safe-bash/command-chunks/chunk-E47FFS5M.js";
+import "../safe-bash/command-chunks/chunk-QX3WKBLI.js";
+import "../safe-bash/command-chunks/chunk-GNPPPYDF.js";
 import "../safe-bash/command-chunks/chunk-VMA3NJXU.js";
 import "../safe-bash/command-chunks/chunk-65DS4IOB.js";
 export {

package/dist/safe-bash-command-llm/elevenlabs.js

@@ -1,8 +1,8 @@
 import {
   createElevenLabsProvider
-} from "../safe-bash/command-chunks/chunk-O2XP6HQN.js";
+} from "../safe-bash/command-chunks/chunk-ZBGP6XEG.js";
 import "../safe-bash/command-chunks/chunk-OE4JR6WJ.js";
-import "../safe-bash/command-chunks/chunk-4SISSPIH.js";
+import "../safe-bash/command-chunks/chunk-HLRLMFMS.js";
 import "../safe-bash/command-chunks/chunk-VMA3NJXU.js";
 import "../safe-bash/command-chunks/chunk-65DS4IOB.js";
 export {

package/dist/safe-bash-command-llm/fragment-loaders.d.ts

@@ -0,0 +1,29 @@
+import type { CommandContext, FileSystem } from "../safe-bash-contracts/index.js";
+import type { LlmInputSource } from './types.js';
+import type { LlmFragmentInputSource } from './fragments.js';
+export type LlmLoadedFragment = {
+    readonly type: 'text';
+    readonly source: LlmFragmentInputSource;
+} | {
+    readonly type: 'attachment';
+    readonly source: LlmInputSource;
+    readonly mimeType: string;
+    readonly id?: string;
+};
+export interface LlmFragmentLoaderContext {
+    readonly fs: FileSystem;
+    readonly cwd: string;
+    readonly signal: AbortSignal;
+    /** Remaining command admission. The host must bound acquisition before yielding. */
+    readonly maxBytes: number;
+    readonly capabilities?: CommandContext['capabilities'];
+}
+/** Acquire lazily. A yielded source is borrowed until the next pull or return. */
+export type LlmFragmentLoader = ((value: string, context: LlmFragmentLoaderContext) => AsyncIterable<LlmLoadedFragment>) & {
+    readonly description?: string;
+};
+export declare function getLlmFragmentPrefix(reference: string): string | undefined;
+/** Preserve registration order and the reference's collision suffixes. */
+export declare function createLlmFragmentLoaders(entries: Iterable<readonly [string, LlmFragmentLoader]>): ReadonlyMap<string, LlmFragmentLoader>;
+/** Resolve an injected loader without ambient plugin imports or history. */
+export declare function loadLlmPluginFragments(reference: string, loaders: ReadonlyMap<string, LlmFragmentLoader>, context: LlmFragmentLoaderContext, allowAttachments?: boolean): AsyncGenerator<LlmLoadedFragment>;

package/dist/safe-bash-command-llm/fragments.d.ts

@@ -0,0 +1,24 @@
+import type { FileSystem } from "../safe-bash-contracts/index.js";
+import type { LlmInputSource } from "./types.js";
+export interface LlmFragmentInputSource extends LlmInputSource {
+    /** Override the composition default for this source, such as HTTP text. */
+    readonly normalizeNewlines?: boolean;
+}
+export interface LlmFragmentSourceOptions {
+    readonly fs: FileSystem;
+    readonly directory: string;
+    readonly signal: AbortSignal;
+    /** Sources are acquired sequentially and disposed after composition. */
+    readonly fragments: AsyncIterable<LlmFragmentInputSource>;
+    /** Already-admitted prompt or system text appended after the fragments. */
+    readonly tail?: LlmInputSource;
+    readonly system?: boolean;
+    /** Match Python text-file universal newlines on fragments (not the tail). */
+    readonly normalizeNewlines?: boolean;
+    /** Charge raw fragment bytes before retaining them. Tail bytes are excluded. */
+    readonly admitBytes?: (size: number, source: LlmFragmentInputSource) => void;
+    readonly admitSeparator?: (size: number) => void;
+}
+/** Compose reference LLM fragments without history or whole-input buffering.
+ * Working memory is bounded; retained content uses the caller's filesystem. */
+export declare function createLlmFragmentSource(options: LlmFragmentSourceOptions): Promise<LlmInputSource>;

package/dist/safe-bash-command-llm/import-json-document.d.ts

@@ -1,5 +1,8 @@
 import type { FileSystem } from "../safe-bash-contracts/index.js";
 type NodeType = 'object' | 'array' | 'string' | 'number' | 'boolean' | 'null';
+export declare class EmptyJsonDocumentError extends Error {
+    constructor();
+}
 export interface EmbeddingJsonNode {
     readonly id: number;
     readonly type: NodeType;
@@ -18,8 +21,8 @@ export interface EmbeddingJsonDocument {
     text(node: EmbeddingJsonNode): AsyncIterable<string>;
     points(node: EmbeddingJsonNode): AsyncIterable<readonly number[]>;
 }
-/** Validate a complete JSON document before exposing it. Retained code-point payloads
- * preserve individual surrogate code points until the consuming encoder handles them. */
+/** Validate a complete JSON document before exposing caller-backed records.
+ * Code points remain distinct even when their JavaScript strings look alike. */
 export declare function withEmbeddingJsonDocument<T>(options: {
     fs: FileSystem;
     directory: string;

package/dist/safe-bash-command-llm/index.d.ts

@@ -1,4 +1,7 @@
 export { serializeLlmTokenUsage } from "./usage.js";
+export { createLlmToolRegistry, selectLlmTools, type LlmRegisteredTool, type LlmToolLoader, type LlmToolboxDescription, type LlmPluginQuery, type LlmPluginInfo } from './tool-registry.js';
+export { executeLlmToolCalls, LlmCancelToolCall, type LlmToolContext, type LlmToolOutput, type LlmExecutableTool, type LlmToolExecutionResult, type LlmToolExecutionOptions } from './tool-execution.js';
+export type { LlmTool, LlmToolCall, LlmMessage } from "./types.js";
 export { createLlmInputBudget, type LlmInputLimits } from "./input-budget.js";
 export * from "./provider-serialization.js";
 export { llmReferenceVersion, createLlmCommand, createLlmCommands, llmCommands } from "./command.js";
@@ -20,3 +23,7 @@ export { openAiChatOptions } from "./openai-chat-options.js";
 export { createLlmUrlSource, type LlmUrlSourceOptions } from './url-source.js';
 export { resolveUrlAttachment as resolveLlmUrlAttachment } from './url-attachment.js';
 export { getLlmAttachmentUrlId } from './attachment-id.js';
+export { createLlmFragmentSource, type LlmFragmentInputSource, type LlmFragmentSourceOptions } from "./fragments.js";
+export { createLlmUrlFragmentSource } from "./url-fragment-source.js";
+export { streamLlmToolChain, type LlmToolChainOptions } from "./tool-chain.js";
+export { createLlmFragmentLoaders, loadLlmPluginFragments, getLlmFragmentPrefix, type LlmFragmentLoader, type LlmFragmentLoaderContext, type LlmLoadedFragment } from "./fragment-loaders.js";