@maycombhq/atticus 0.0.0-stage → 2.0.0-beta.11

package/dist/chunk-EZSZFI54.js

@@ -0,0 +1,531 @@
+import { createRequire as __atticusCreateRequire } from "node:module";
+import { fileURLToPath as __atticusFileURLToPath } from "node:url";
+var require = __atticusCreateRequire(import.meta.url);
+var __atticusUrl = (p) => new URL("./" + p, import.meta.url).href;
+var __atticusPath = (p) => __atticusFileURLToPath(__atticusUrl(p));
+import {
+  quantize
+} from "./chunk-BKFGGJKA.js";
+import {
+  Runtime,
+  external_exports,
+  orgPackDirOf,
+  readTarGz,
+  require_dist,
+  retrievalOf
+} from "./chunk-JMOLB347.js";
+import "./chunk-MMB2OHUY.js";
+import {
+  __toESM
+} from "./chunk-TV45EUGM.js";
+
+// packages/embed/dist/wordpiece.js
+function wordPieceConfig(json) {
+  const t = json;
+  if (t?.model?.type !== "WordPiece")
+    throw new Error(`the tokenizer is ${t?.model?.type ?? "unknown"}, not WordPiece`);
+  const lowercase = t.normalizer?.lowercase ?? true;
+  return {
+    vocab: new Map(Object.entries(t.model.vocab)),
+    unk: t.model.unk_token ?? "[UNK]",
+    prefix: t.model.continuing_subword_prefix ?? "##",
+    maxCharsPerWord: t.model.max_input_chars_per_word ?? 100,
+    lowercase,
+    // BertNormalizer: strip_accents null means "as lowercase".
+    stripAccents: t.normalizer?.strip_accents ?? lowercase,
+    cls: "[CLS]",
+    sep: "[SEP]"
+  };
+}
+var isControl = (c) => {
+  if (c === "	" || c === "\n" || c === "\r")
+    return false;
+  return new RegExp("\\p{Cc}|\\p{Cf}", "u").test(c);
+};
+var isWhite = (c) => /\s/u.test(c);
+var isPunct = (c) => {
+  const cp = c.codePointAt(0);
+  if (cp >= 33 && cp <= 47 || cp >= 58 && cp <= 64 || cp >= 91 && cp <= 96 || cp >= 123 && cp <= 126)
+    return true;
+  return new RegExp("\\p{P}", "u").test(c);
+};
+var isCjk = (cp) => cp >= 19968 && cp <= 40959 || cp >= 13312 && cp <= 19903 || cp >= 131072 && cp <= 173791 || cp >= 173824 && cp <= 177983 || cp >= 177984 && cp <= 178207 || cp >= 178208 && cp <= 183983 || cp >= 63744 && cp <= 64255 || cp >= 194560 && cp <= 195103;
+function normalize(text, c) {
+  let out = "";
+  for (const ch of text) {
+    const cp = ch.codePointAt(0);
+    if (cp === 0 || cp === 65533 || isControl(ch))
+      continue;
+    if (isWhite(ch))
+      out += " ";
+    else if (isCjk(cp))
+      out += ` ${ch} `;
+    else
+      out += ch;
+  }
+  if (c.lowercase)
+    out = out.toLowerCase();
+  if (c.stripAccents)
+    out = out.normalize("NFD").replace(new RegExp("\\p{Mn}", "gu"), "");
+  return out;
+}
+function preTokenize(text) {
+  const out = [];
+  let cur = "";
+  for (const ch of text) {
+    if (isWhite(ch)) {
+      if (cur)
+        out.push(cur);
+      cur = "";
+    } else if (isPunct(ch)) {
+      if (cur)
+        out.push(cur);
+      out.push(ch);
+      cur = "";
+    } else
+      cur += ch;
+  }
+  if (cur)
+    out.push(cur);
+  return out;
+}
+function wordPieces(word, c) {
+  const chars = [...word];
+  if (chars.length > c.maxCharsPerWord)
+    return [c.unk];
+  const out = [];
+  let start = 0;
+  while (start < chars.length) {
+    let end = chars.length;
+    let found;
+    while (start < end) {
+      const sub = `${start > 0 ? c.prefix : ""}${chars.slice(start, end).join("")}`;
+      if (c.vocab.has(sub)) {
+        found = sub;
+        break;
+      }
+      end--;
+    }
+    if (!found)
+      return [c.unk];
+    out.push(found);
+    start = end;
+  }
+  return out;
+}
+function encode(text, c, maxTokens) {
+  const ids = [c.vocab.get(c.cls)];
+  const unk = c.vocab.get(c.unk);
+  outer: for (const w of preTokenize(normalize(text, c)))
+    for (const p of wordPieces(w, c)) {
+      if (ids.length >= maxTokens - 1)
+        break outer;
+      ids.push(c.vocab.get(p) ?? unk);
+    }
+  ids.push(c.vocab.get(c.sep));
+  return ids;
+}
+
+// packages/embed/dist/model.js
+var import_yaml = __toESM(require_dist(), 1);
+import { createHash } from "node:crypto";
+import { createReadStream, existsSync } from "node:fs";
+import { readFile, stat } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { dirname, join, sep } from "node:path";
+import { pathToFileURL } from "node:url";
+var __atticus_import_meta_url = __atticusUrl("packages/embed/dist/model.js");
+var MODEL_MANIFEST = "atticus-model.yaml";
+var ModelManifest = external_exports.strictObject({
+  schemaVersion: external_exports.literal(1),
+  /** A short name, e.g. snowflake-arctic-embed-xs. */
+  name: external_exports.string().regex(/^[a-z0-9][a-z0-9._-]*$/),
+  /** fp32 only: an int8 model's vectors change with the batch's shape and the backend. */
+  dtype: external_exports.literal("fp32"),
+  /** The model file, fp32 ONNX, in the folder. */
+  model: external_exports.string().min(1).default("model.onnx"),
+  sha256: external_exports.string().regex(/^[0-9a-f]{64}$/),
+  tokenizer: external_exports.string().min(1).default("tokenizer.json"),
+  dim: external_exports.number().int().positive(),
+  maxTokens: external_exports.number().int().positive(),
+  /** cls: the first token's vector; mean: the average over the tokens. */
+  pooling: external_exports.enum(["cls", "mean"]),
+  /** Said before a query (some retrieval models are trained with one). */
+  queryPrefix: external_exports.string().default(""),
+  documentPrefix: external_exports.string().default(""),
+  /** The cosine below which a node is not similar, for this model (calibrated on the eval). */
+  similarityFloor: external_exports.number().min(-1).max(1).optional(),
+  /** The model's output to pool. */
+  output: external_exports.string().default("last_hidden_state"),
+  licence: external_exports.string().min(1),
+  /** Where the weights were taken from, for the SBOM. */
+  source: external_exports.string().min(1)
+});
+async function readModelManifest(dir) {
+  const text = await readFile(join(dir, MODEL_MANIFEST), "utf8");
+  return ModelManifest.parse((0, import_yaml.parse)(text));
+}
+function fileSha256(file) {
+  return new Promise((resolve2, reject) => {
+    const h = createHash("sha256");
+    createReadStream(file).on("data", (b) => h.update(b)).on("error", reject).on("end", () => resolve2(h.digest("hex")));
+  });
+}
+var runtimePackages = (kind) => kind === "native" ? [`@maycombhq/onnxruntime-${process.platform}-${process.arch}`, "onnxruntime-node"] : ["@maycombhq/onnxruntime-wasm", "onnxruntime-web"];
+function packageDir(req, name) {
+  try {
+    return dirname(req.resolve(`${name}/package.json`));
+  } catch {
+    try {
+      const entry = req.resolve(name);
+      const at = entry.lastIndexOf(`${sep}${name.split("/").join(sep)}${sep}`);
+      return at < 0 ? void 0 : entry.slice(0, at + name.length + 1);
+    } catch {
+      return void 0;
+    }
+  }
+}
+async function loadOrt(kind, runtimeDir) {
+  const req = createRequire(runtimeDir ? join(runtimeDir, "noop.js") : __atticus_import_meta_url);
+  for (const name of runtimePackages(kind)) {
+    const dir = packageDir(req, name);
+    if (!dir)
+      continue;
+    const file = join(dir, "dist", kind === "native" ? "index.js" : "ort.wasm.min.mjs");
+    if (!existsSync(file))
+      continue;
+    const m = kind === "native" ? createRequire(file)(file) : await import(pathToFileURL(file).href);
+    return { ort: m.default ?? m, dir };
+  }
+  throw new Error(`no ${kind} ONNX runtime is installed (${runtimePackages(kind).join(" or ")})`);
+}
+var OnnxEmbedder = class _OnnxEmbedder {
+  dir;
+  manifest;
+  o;
+  id;
+  dim;
+  maxTokens;
+  backend = "not loaded";
+  floor;
+  session;
+  constructor(dir, manifest, o = {}) {
+    this.dir = dir;
+    this.manifest = manifest;
+    this.o = o;
+    this.id = `${manifest.name}@${manifest.sha256.slice(0, 12)}`;
+    this.dim = manifest.dim;
+    this.maxTokens = manifest.maxTokens;
+    if (manifest.similarityFloor !== void 0)
+      this.floor = manifest.similarityFloor;
+  }
+  static async open(dir, o = {}) {
+    return new _OnnxEmbedder(dir, await readModelManifest(dir), o);
+  }
+  /** The model file is there and is the one the manifest names (its sha256). */
+  async check() {
+    const file = join(this.dir, this.manifest.model);
+    if (!existsSync(file))
+      return { ok: false, problem: `${file} isn't there` };
+    const sha = await fileSha256(file);
+    return sha === this.manifest.sha256 ? { ok: true } : {
+      ok: false,
+      problem: `${file} has sha256 ${sha}, not the ${this.manifest.sha256} its manifest names`
+    };
+  }
+  load() {
+    this.session ??= (async () => {
+      const c = await this.check();
+      if (!c.ok)
+        throw new Error(c.problem);
+      const tok = wordPieceConfig(JSON.parse(await readFile(join(this.dir, this.manifest.tokenizer), "utf8")));
+      const file = join(this.dir, this.manifest.model);
+      let loaded;
+      let s;
+      if (this.o.backend !== "wasm") {
+        try {
+          loaded = await loadOrt("native", this.o.runtimeDir);
+          s = await loaded.ort.InferenceSession.create(file, {
+            executionProviders: ["cpu"],
+            graphOptimizationLevel: "all",
+            intraOpNumThreads: 1
+          });
+          this.backend = "native";
+        } catch {
+          s = void 0;
+        }
+      }
+      if (!s) {
+        loaded = await loadOrt("wasm", this.o.runtimeDir);
+        if (loaded.ort.env?.wasm) {
+          loaded.ort.env.wasm.numThreads = 1;
+          loaded.ort.env.wasm.wasmBinary = await readFile(join(loaded.dir, "dist", "ort-wasm-simd-threaded.wasm"));
+        }
+        s = await loaded.ort.InferenceSession.create(await readFile(file), {
+          executionProviders: ["wasm"]
+        });
+        this.backend = "wasm";
+      }
+      return { s, ort: loaded.ort, tok };
+    })();
+    return this.session;
+  }
+  async embed(texts, kind) {
+    const { s, ort, tok } = await this.load();
+    const out = [];
+    const prefix = kind === "query" ? this.manifest.queryPrefix : this.manifest.documentPrefix;
+    for (const t of texts) {
+      const ids = encode(`${prefix}${t}`, tok, this.maxTokens);
+      const n = ids.length;
+      const feeds = {
+        input_ids: new ort.Tensor("int64", BigInt64Array.from(ids.map(BigInt)), [1, n]),
+        attention_mask: new ort.Tensor("int64", new BigInt64Array(n).fill(1n), [1, n])
+      };
+      if (s.inputNames.includes("token_type_ids"))
+        feeds.token_type_ids = new ort.Tensor("int64", new BigInt64Array(n), [1, n]);
+      const r = await s.run(feeds);
+      const h = r[this.manifest.output];
+      if (!h)
+        throw new Error(`the model has no output ${this.manifest.output}`);
+      const d = h.dims.at(-1);
+      const v = new Float64Array(d);
+      if (this.manifest.pooling === "cls")
+        for (let j = 0; j < d; j++)
+          v[j] = h.data[j];
+      else {
+        for (let i = 0; i < n; i++)
+          for (let j = 0; j < d; j++)
+            v[j] = v[j] + h.data[i * d + j];
+        for (let j = 0; j < d; j++)
+          v[j] = v[j] / n;
+      }
+      out.push(quantize(v));
+    }
+    return out;
+  }
+};
+async function modelInfo(dir) {
+  try {
+    const manifest = await readModelManifest(dir);
+    const st = await stat(join(dir, manifest.model)).catch(() => void 0);
+    if (!st)
+      return { ok: false, problem: `${join(dir, manifest.model)} isn't there` };
+    return { ok: true, manifest, bytes: st.size };
+  } catch (e) {
+    return {
+      ok: false,
+      problem: `${join(dir, MODEL_MANIFEST)}: ${e.message.split("\n")[0]}`
+    };
+  }
+}
+
+// packages/embed/dist/resolve.js
+var import_yaml2 = __toESM(require_dist(), 1);
+import { existsSync as existsSync2 } from "node:fs";
+import { readFile as readFile2 } from "node:fs/promises";
+import { createRequire as createRequire2 } from "node:module";
+import { isAbsolute, join as join2, resolve } from "node:path";
+var __atticus_import_meta_url2 = __atticusUrl("packages/embed/dist/resolve.js");
+function bundledModelDir(runtimeDir) {
+  try {
+    const req = createRequire2(runtimeDir ? join2(runtimeDir, "noop.js") : __atticus_import_meta_url2);
+    return join2(req.resolve("@maycombhq/embed-model/package.json"), "..");
+  } catch {
+    return void 0;
+  }
+}
+function installedModelDir() {
+  const dir = bundledModelDir();
+  return dir && existsSync2(join2(dir, "model.onnx")) ? dir : void 0;
+}
+var embeddingsOffByEnv = () => process.env.ATTICUS_EMBEDDINGS === "off";
+async function settingsOf(root) {
+  const text = await readFile2(join2(orgPackDirOf(root), "runtime.yaml"), "utf8").catch(() => void 0);
+  const raw = text ? (0, import_yaml2.parse)(text) : void 0;
+  const r = raw === void 0 ? void 0 : Runtime.safeParse(raw);
+  const e = retrievalOf(r?.success ? r.data : void 0).embeddings;
+  const said = raw?.retrieval?.embeddings?.model;
+  return {
+    model: e.model,
+    ...e.runtime ? { runtime: e.runtime } : {},
+    configured: r?.success === true && said !== void 0
+  };
+}
+async function embeddingState(root) {
+  const s = await settingsOf(root);
+  const configured = s.configured;
+  if (embeddingsOffByEnv())
+    return { setting: "off", configured, problem: "ATTICUS_EMBEDDINGS is off for this process" };
+  if (s.model === "off")
+    return { setting: "off", configured, problem: "retrieval.embeddings.model is off" };
+  const base = orgPackDirOf(root);
+  const runtimeDir = s.runtime ? resolve(base, s.runtime) : void 0;
+  const dir = s.model === "bundled" ? bundledModelDir(runtimeDir) : isAbsolute(s.model) ? s.model : resolve(base, s.model);
+  if (!dir)
+    return {
+      setting: s.model,
+      configured,
+      problem: "the bundled model (@maycombhq/embed-model) isn't installed"
+    };
+  const info = await modelInfo(dir);
+  if (!info.ok)
+    return { setting: s.model, configured, dir, problem: info.problem };
+  return {
+    setting: s.model,
+    configured,
+    dir,
+    id: `${info.manifest.name}@${info.manifest.sha256.slice(0, 12)}`,
+    bytes: info.bytes,
+    licence: info.manifest.licence
+  };
+}
+async function runtimeAvailable(root) {
+  const s = await settingsOf(root);
+  const dir = s.runtime ? resolve(orgPackDirOf(root), s.runtime) : void 0;
+  const req = createRequire2(dir ? join2(dir, "noop.js") : __atticus_import_meta_url2);
+  for (const kind of ["native", "wasm"])
+    for (const name of runtimePackages(kind)) {
+      const d = packageDir(req, name);
+      if (d && existsSync2(join2(d, "dist", kind === "native" ? "index.js" : "ort.wasm.min.mjs")))
+        return kind;
+    }
+  return void 0;
+}
+async function embedderFor(root) {
+  const state = await embeddingState(root);
+  if (!state.dir || state.problem)
+    return { state };
+  const s = await settingsOf(root);
+  const runtimeDir = s.runtime ? resolve(orgPackDirOf(root), s.runtime) : void 0;
+  try {
+    return {
+      embedder: await OnnxEmbedder.open(state.dir, runtimeDir ? { runtimeDir } : {}),
+      state
+    };
+  } catch (e) {
+    return {
+      state: {
+        ...state,
+        problem: `the model in ${state.dir} couldn't be opened: ${e.message.split("\n")[0]}`
+      }
+    };
+  }
+}
+
+// packages/embed/dist/pinned.js
+import { createHash as createHash2 } from "node:crypto";
+import { mkdir, readFile as readFile3, rename, writeFile } from "node:fs/promises";
+import { dirname as dirname2, join as join3 } from "node:path";
+var sha256Of = (b) => createHash2("sha256").update(b).digest("hex");
+var PinError = class extends Error {
+  name = "PinError";
+};
+function checkPinned(bytes, sha256, what) {
+  const got = sha256Of(bytes);
+  if (got !== sha256)
+    throw new PinError(`${what} has sha256 ${got}, not the pinned ${sha256}: refused`);
+}
+var httpGet = async (url) => {
+  const r = await fetch(url, { redirect: "follow" });
+  if (!r.ok)
+    throw new PinError(`${url}: HTTP ${r.status}`);
+  return new Uint8Array(await r.arrayBuffer());
+};
+async function fetchSource(src, o) {
+  const file = join3(o.cacheDir, src.sha256);
+  const kept = await readFile3(file).catch(() => void 0);
+  if (kept && sha256Of(kept) === src.sha256)
+    return { bytes: kept, cached: true };
+  if (o.offline)
+    throw new PinError(`${src.url} isn't in the cache and this run may not download`);
+  const bytes = await (o.get ?? httpGet)(src.url);
+  checkPinned(bytes, src.sha256, src.url);
+  await mkdir(o.cacheDir, { recursive: true });
+  await writeFile(`${file}.part`, bytes);
+  await rename(`${file}.part`, file);
+  return { bytes, cached: false };
+}
+async function materialize(dir, pins, o) {
+  const downloaded = [];
+  const bySource = /* @__PURE__ */ new Map();
+  const sourceBytes = async (name) => {
+    let b = bySource.get(name);
+    if (b)
+      return b;
+    const src = pins.sources[name];
+    if (!src)
+      throw new PinError(`pins.json names no source ${name}`);
+    const r = await fetchSource(src, o);
+    if (!r.cached)
+      downloaded.push(src.url);
+    bySource.set(name, b = r.bytes);
+    return b;
+  };
+  const archives = /* @__PURE__ */ new Map();
+  const written = [];
+  for (const f of pins.files) {
+    if (f.path.split(/[\\/]/).includes(".."))
+      throw new PinError(`${f.path} leaves the package`);
+    let bytes;
+    if (f.entry) {
+      let a = archives.get(f.source);
+      if (!a)
+        archives.set(f.source, a = readTarGz(await sourceBytes(f.source), {
+          maxEntries: 2e4,
+          maxEntryBytes: 256 * 1024 * 1024,
+          maxTotalBytes: 1024 * 1024 * 1024
+        }));
+      const e = a.read(f.entry);
+      if (!e)
+        throw new PinError(`${f.source} has no ${f.entry}`);
+      bytes = e;
+    } else
+      bytes = await sourceBytes(f.source);
+    checkPinned(bytes, f.sha256, f.path);
+    const out = join3(dir, ...f.path.split("/"));
+    const was = await readFile3(out).catch(() => void 0);
+    if (!was || sha256Of(was) !== f.sha256) {
+      await mkdir(dirname2(out), { recursive: true });
+      await writeFile(out, bytes);
+    }
+    written.push(f.path);
+  }
+  return { written, downloaded };
+}
+async function checkMaterialized(dir, pins) {
+  const bad = [];
+  for (const f of pins.files) {
+    const b = await readFile3(join3(dir, ...f.path.split("/"))).catch(() => void 0);
+    if (!b || sha256Of(b) !== f.sha256)
+      bad.push(f.path);
+  }
+  return bad;
+}
+export {
+  MODEL_MANIFEST,
+  ModelManifest,
+  OnnxEmbedder,
+  PinError,
+  bundledModelDir,
+  checkMaterialized,
+  checkPinned,
+  embedderFor,
+  embeddingState,
+  embeddingsOffByEnv,
+  encode,
+  fetchSource,
+  fileSha256,
+  httpGet,
+  installedModelDir,
+  materialize,
+  modelInfo,
+  normalize,
+  packageDir,
+  preTokenize,
+  readModelManifest,
+  runtimeAvailable,
+  runtimePackages,
+  sha256Of,
+  wordPieceConfig,
+  wordPieces
+};