defuss-vectorsearch 0.0.0-stage → 0.1.0

package/dist/client.d.mts

@@ -0,0 +1,15 @@
+import { B as BaseEmbeddingRuntime } from './core-DnBchjh2.mjs';
+import { E as EmbedderInitOptions } from './winzling-embedder-Dqq64fVb.mjs';
+export { B as BuildTurboQuantIndexOptions, f as EmbedOptions, g as Embedder, h as EmbeddingDType, i as EmbeddingDevice, j as ExactSearchMulticoreOptions, F as FeatureExtractorLike, M as ModelCacheClearFileResult, k as ModelCacheClearResult, l as ModelCacheFileStatus, m as ModelCacheInspectionResult, n as ModelCacheLocation, o as ModelCacheOptions, q as ModelLoadOptions, r as ModelPrefetchFile, s as ModelPrefetchOptions, t as ModelPrefetchResult, u as ModelSourceKind, O as OpenAICompatibleEmbeddingEndpointOptions, P as PoolingStrategy, Q as QueryEmbedOptions, v as QueryInstructionPreset, R as ResolvedModelSource, S as SearchHit, w as SearchResult, x as TensorLike, y as TransformersEnvLike, z as TransformersModuleLike, T as TurboQuantRerankResult, b as TurboQuantRerankedSearchHit, c as TurboQuantSearchIndex, A as Vector, V as Vectors } from './winzling-embedder-Dqq64fVb.mjs';
+export { DEFAULT_MODEL_ID, DEFAULT_MODEL_REMOTE_HOST, DEFAULT_MODEL_REMOTE_PATH_TEMPLATE, DEFAULT_QUERY_INSTRUCTION_PRESET, DEFAULT_TRANSFORMERS_CACHE_NAME, LEGACY_MODEL_ID, QUERY_INSTRUCTION_PRESETS, buildNodeCacheKey, buildRemoteModelFileUrl, clearModelCache, formatInstructionQuery, getRequiredModelFiles, inspectModelCache, pathJoin, prefetchModel, resolveModelSource, resolveQueryInstruction } from './shared.mjs';
+export { attachRecords, searchTopKMulticore } from './vector-search.mjs';
+export { buildTurboQuantIndex, rerankSearchHits, scoreTurboQuantIndex, searchTurboQuantIndex, searchTurboQuantIndexRerank, selfCheckTurboQuant } from './turboquant.mjs';
+export { S as SUPPORTED_MODELS, W as WINZLING_FILES, a as WINZLING_MODEL_ID, b as WINZLING_PROFILE, c as WINZLING_REVISION } from './model-profiles-BDS-uZ4e.mjs';
+export { d as dotProduct, b as dotProductJS, n as normalizeVector, a as normalizeVectors, s as searchTopK, t as toFloat32, c as topKFromScores } from './vector-math-C29mf_rv.mjs';
+
+declare class DefussEmbeddingClient extends BaseEmbeddingRuntime {
+    constructor(options?: EmbedderInitOptions);
+}
+declare const createEmbeddingClient: (options?: EmbedderInitOptions) => DefussEmbeddingClient;
+
+export { BaseEmbeddingRuntime, DefussEmbeddingClient, EmbedderInitOptions, createEmbeddingClient };

package/dist/client.mjs

@@ -0,0 +1,23 @@
+import { B as BaseEmbeddingRuntime } from './core-DSMVtqZ6.mjs';
+export { D as DEFAULT_MODEL_ID, a as DEFAULT_MODEL_REMOTE_HOST, b as DEFAULT_MODEL_REMOTE_PATH_TEMPLATE, c as DEFAULT_TRANSFORMERS_CACHE_NAME, L as LEGACY_MODEL_ID, S as SUPPORTED_MODELS, W as WINZLING_FILES, d as WINZLING_MODEL_ID, e as WINZLING_PROFILE, f as WINZLING_REVISION, g as buildNodeCacheKey, h as buildRemoteModelFileUrl, i as getRequiredModelFiles, p as pathJoin, r as resolveModelSource } from './model-cache.browser-BVWrcI3Y.mjs';
+export { D as DEFAULT_QUERY_INSTRUCTION_PRESET, Q as QUERY_INSTRUCTION_PRESETS, c as clearModelCache, f as formatInstructionQuery, i as inspectModelCache, p as prefetchModel, r as resolveQueryInstruction } from './prompts-BbVgTLA6.mjs';
+export { attachRecords, searchTopKMulticore } from './vector-search.mjs';
+export { buildTurboQuantIndex, rerankSearchHits, scoreTurboQuantIndex, searchTurboQuantIndex, searchTurboQuantIndexRerank, selfCheckTurboQuant } from './turboquant.mjs';
+export { d as dotProduct, a as dotProductJS, n as normalizeVector, b as normalizeVectors, s as searchTopK, t as toFloat32, c as topKFromScores } from './vector-math-DwdA_xwv.mjs';
+import './onnx.mjs';
+import './model-cache-qd7znbJ5.mjs';
+import './winzling-embedder-CthXBDRS.mjs';
+import '@huggingface/tokenizers';
+import 'defuss-db';
+import 'defuss-db/client.js';
+
+class DefussEmbeddingClient extends BaseEmbeddingRuntime {
+  constructor(options = {}) {
+    super(options, { allowLocalModels: false, useBrowserCache: true });
+  }
+}
+const createEmbeddingClient = (options = {}) => {
+  return new DefussEmbeddingClient(options);
+};
+
+export { BaseEmbeddingRuntime, DefussEmbeddingClient, createEmbeddingClient };

package/dist/core-DSMVtqZ6.mjs

@@ -0,0 +1,353 @@
+import { d as WINZLING_MODEL_ID, D as DEFAULT_MODEL_ID, r as resolveModelSource, f as WINZLING_REVISION, e as WINZLING_PROFILE } from './model-cache.browser-BVWrcI3Y.mjs';
+import { p as prefetchModel, i as inspectModelCache, c as clearModelCache, f as formatInstructionQuery } from './prompts-BbVgTLA6.mjs';
+import { createWinzlingExtractor } from './onnx.mjs';
+import { n as normalizeVector } from './vector-math-DwdA_xwv.mjs';
+
+const isRecord = (value) => {
+  return typeof value === "object" && value !== null;
+};
+const resolveFetch = (options) => {
+  const fetchImpl = options.fetch ?? globalThis.fetch;
+  if (typeof fetchImpl !== "function") {
+    throw new Error(
+      "OpenAI-compatible embedding endpoint mode requires a fetch implementation."
+    );
+  }
+  return fetchImpl;
+};
+const resolveOpenAICompatibleEmbeddingsUrl = (options) => {
+  if (options.endpoint) {
+    return new URL(options.endpoint).toString();
+  }
+  if (options.baseUrl) {
+    const baseUrl = options.baseUrl.endsWith("/") ? options.baseUrl : `${options.baseUrl}/`;
+    return new URL("embeddings", baseUrl).toString();
+  }
+  throw new Error(
+    "openAICompatible requires either endpoint or baseUrl to resolve the embeddings URL."
+  );
+};
+const buildRequestHeaders = (options) => {
+  const headers = new Headers();
+  headers.set("Accept", "application/json");
+  headers.set("Content-Type", "application/json");
+  for (const [key, value] of Object.entries(options.headers ?? {})) {
+    headers.set(key, value);
+  }
+  if (options.apiKey && !headers.has("Authorization")) {
+    headers.set("Authorization", `Bearer ${options.apiKey}`);
+  }
+  return headers;
+};
+const parseEmbeddingVector = (value, index) => {
+  if (!Array.isArray(value)) {
+    throw new Error(`Invalid embedding payload at index ${index}: expected an array.`);
+  }
+  const embedding = new Float32Array(value.length);
+  for (let offset = 0; offset < value.length; offset++) {
+    const numeric = Number(value[offset]);
+    if (!Number.isFinite(numeric)) {
+      throw new Error(
+        `Invalid embedding payload at index ${index}: value ${offset} is not numeric.`
+      );
+    }
+    embedding[offset] = numeric;
+  }
+  return embedding;
+};
+const parseEmbeddingResponse = (payload) => {
+  if (!isRecord(payload) || !Array.isArray(payload.data)) {
+    throw new Error("Invalid OpenAI-compatible embeddings response: expected a data array.");
+  }
+  const items = payload.data.map((entry, index) => {
+    if (Array.isArray(entry)) {
+      return {
+        index,
+        embedding: parseEmbeddingVector(entry, index)
+      };
+    }
+    if (!isRecord(entry)) {
+      throw new Error(`Invalid embedding payload at index ${index}: expected an object.`);
+    }
+    return {
+      index: typeof entry.index === "number" ? entry.index : index,
+      embedding: parseEmbeddingVector(entry.embedding, index)
+    };
+  });
+  items.sort((a, b) => a.index - b.index);
+  return items.map((item) => item.embedding);
+};
+const fetchOpenAICompatibleEmbeddings = async (input, model, options) => {
+  const fetchImpl = resolveFetch(options);
+  const endpointUrl = resolveOpenAICompatibleEmbeddingsUrl(options);
+  const body = {
+    model,
+    input,
+    ...options.encodingFormat !== void 0 ? { encoding_format: options.encodingFormat } : {},
+    ...options.dimensions !== void 0 ? { dimensions: options.dimensions } : {},
+    ...options.user !== void 0 ? { user: options.user } : {},
+    ...options.extraBody
+  };
+  const response = await fetchImpl(endpointUrl, {
+    method: "POST",
+    headers: buildRequestHeaders(options),
+    body: JSON.stringify(body)
+  });
+  const rawBody = await response.text();
+  if (!response.ok) {
+    throw new Error(
+      `Failed to fetch embeddings from ${endpointUrl} (${response.status}): ${rawBody || response.statusText}`
+    );
+  }
+  let payload;
+  try {
+    payload = rawBody.length > 0 ? JSON.parse(rawBody) : {};
+  } catch (error) {
+    throw new Error(
+      `Invalid JSON from OpenAI-compatible embeddings endpoint ${endpointUrl}: ${error.message}`
+    );
+  }
+  return parseEmbeddingResponse(payload);
+};
+
+const DEFAULT_MODEL = DEFAULT_MODEL_ID;
+const DEFAULT_DTYPE = "fp32";
+const DEFAULT_POOLING = "last_token";
+const DEFAULT_NORMALIZE = true;
+const getTransformersModule = async () => {
+  return import('@huggingface/transformers');
+};
+const configureEnv = (mod, options, defaults) => {
+  mod.env.allowRemoteModels = options.allowRemoteModels ?? true;
+  mod.env.allowLocalModels = options.allowLocalModels ?? defaults.allowLocalModels;
+  if (options.localModelPath !== void 0) mod.env.localModelPath = options.localModelPath;
+  if (options.cacheDir !== void 0) mod.env.cacheDir = options.cacheDir;
+  if (options.useBrowserCache !== void 0) mod.env.useBrowserCache = options.useBrowserCache;
+  else if (defaults.useBrowserCache !== void 0) mod.env.useBrowserCache = defaults.useBrowserCache;
+  if (options.useFSCache !== void 0) mod.env.useFSCache = options.useFSCache;
+  if (options.useFS !== void 0) mod.env.useFS = options.useFS;
+  if (options.logLevel !== void 0) mod.env.logLevel = options.logLevel;
+};
+const splitTensor = (tensor) => {
+  const dims = tensor.dims;
+  if (dims.length !== 2) {
+    throw new Error(`Expected pooled 2D tensor, got dims=${JSON.stringify(dims)}`);
+  }
+  const batch = dims[0];
+  const width = dims[1];
+  const data = tensor.data instanceof Float32Array ? tensor.data : Float32Array.from(tensor.data);
+  const embeddings = [];
+  for (let row = 0; row < batch; row++) {
+    const start = row * width;
+    embeddings[row] = data.slice(start, start + width);
+  }
+  return embeddings;
+};
+class BaseEmbeddingRuntime {
+  model;
+  get dtype() {
+    return this.options.dtype ?? (this.isWinzling() ? "q4" : DEFAULT_DTYPE);
+  }
+  get device() {
+    return this.options.device ?? (this.isWinzling() ? this.options.winzling?.device ?? "wasm" : void 0);
+  }
+  options;
+  runtimeDefaults;
+  resolvedModelSource;
+  requiredFiles;
+  queue = Promise.resolve();
+  closed = false;
+  closing;
+  extractorPromise = null;
+  constructor(options = {}, runtimeDefaults) {
+    this.options = options;
+    this.runtimeDefaults = runtimeDefaults;
+    this.model = options.model ?? options.openAICompatible?.model ?? (options.modelProfile === "winzling" ? WINZLING_MODEL_ID : DEFAULT_MODEL);
+    this.resolvedModelSource = options.openAICompatible ? null : resolveModelSource(this.model, {
+      revision: options.revision ?? (options.modelProfile === "winzling" ? WINZLING_REVISION : void 0)
+    });
+    this.requiredFiles = options.requiredFiles;
+  }
+  isWinzling() {
+    return !this.options.openAICompatible && (this.model === WINZLING_MODEL_ID || this.options.modelProfile === "winzling");
+  }
+  enqueue(operation) {
+    if (this.closed) return Promise.reject(new Error("Embedder has been disposed"));
+    const pending = this.queue.then(operation);
+    this.queue = pending.catch(() => {
+    });
+    return pending;
+  }
+  isOpenAICompatibleMode() {
+    return this.options.openAICompatible !== void 0;
+  }
+  getLocalModelSource(operation) {
+    if (this.resolvedModelSource === null) {
+      throw new Error(
+        `${operation} is unavailable when using an OpenAI-compatible embedding endpoint.`
+      );
+    }
+    return this.resolvedModelSource;
+  }
+  loadModel(urlOrRepoId, options = {}) {
+    return this.enqueue(() => this.loadModelInternal(urlOrRepoId, options));
+  }
+  async loadModelInternal(urlOrRepoId, options) {
+    const previous = await this.extractorPromise;
+    await previous?.dispose?.();
+    this.extractorPromise = null;
+    this.model = urlOrRepoId;
+    if (this.isOpenAICompatibleMode()) {
+      this.extractorPromise = null;
+      return;
+    }
+    this.resolvedModelSource = resolveModelSource(urlOrRepoId, {
+      revision: options.revision ?? this.options.revision ?? (this.isWinzling() ? WINZLING_REVISION : void 0),
+      cacheKey: options.cacheKey,
+      remoteHost: options.remoteHost,
+      remotePathTemplate: options.remotePathTemplate
+    });
+    this.requiredFiles = options.requiredFiles ?? this.options.requiredFiles;
+    this.extractorPromise = null;
+    await this.getExtractor();
+  }
+  async prefetchModel(urlOrRepoId = this.model, options = {}) {
+    const source = this.getLocalModelSource("prefetchModel");
+    return prefetchModel(urlOrRepoId, {
+      revision: options.revision ?? source.revision,
+      dtype: options.dtype ?? this.dtype,
+      cacheKey: options.cacheKey,
+      remoteHost: options.remoteHost,
+      remotePathTemplate: options.remotePathTemplate,
+      cacheDir: options.cacheDir ?? this.options.cacheDir,
+      requiredFiles: options.requiredFiles ?? this.requiredFiles ?? (this.isWinzling() ? WINZLING_PROFILE.requiredFiles : void 0)
+    });
+  }
+  async inspectModelCache(urlOrRepoId = this.model, options = {}) {
+    const source = this.getLocalModelSource("inspectModelCache");
+    return inspectModelCache(urlOrRepoId, {
+      revision: options.revision ?? source.revision,
+      dtype: options.dtype ?? this.dtype,
+      cacheKey: options.cacheKey,
+      remoteHost: options.remoteHost,
+      remotePathTemplate: options.remotePathTemplate,
+      cacheDir: options.cacheDir ?? this.options.cacheDir,
+      requiredFiles: options.requiredFiles ?? this.requiredFiles ?? (this.isWinzling() ? WINZLING_PROFILE.requiredFiles : void 0)
+    });
+  }
+  async clearModelCache(urlOrRepoId = this.model, options = {}) {
+    const source = this.getLocalModelSource("clearModelCache");
+    return clearModelCache(urlOrRepoId, {
+      revision: options.revision ?? source.revision,
+      dtype: options.dtype ?? this.dtype,
+      cacheKey: options.cacheKey,
+      remoteHost: options.remoteHost,
+      remotePathTemplate: options.remotePathTemplate,
+      cacheDir: options.cacheDir ?? this.options.cacheDir,
+      requiredFiles: options.requiredFiles ?? this.requiredFiles ?? (this.isWinzling() ? WINZLING_PROFILE.requiredFiles : void 0)
+    });
+  }
+  embed(input, options = {}) {
+    const snapshot = Array.isArray(input) ? [...input] : input;
+    return this.enqueue(() => this.embedInternal(snapshot, options));
+  }
+  async embedInternal(input, options) {
+    if (Array.isArray(input) && input.length === 0) return [];
+    const shouldNormalize = options.normalize ?? this.options.normalize ?? (this.isWinzling() ? this.options.winzling?.normalize : void 0) ?? DEFAULT_NORMALIZE;
+    if (this.options.openAICompatible) {
+      const embeddings2 = await fetchOpenAICompatibleEmbeddings(
+        input,
+        this.model,
+        this.options.openAICompatible
+      );
+      return shouldNormalize ? embeddings2.map((embedding) => normalizeVector(embedding)) : embeddings2;
+    }
+    const extractor = await this.getExtractor();
+    const output = await extractor(input, {
+      pooling: options.pooling ?? this.options.pooling ?? (this.isWinzling() ? "mean" : DEFAULT_POOLING),
+      normalize: false
+    });
+    const embeddings = splitTensor(output);
+    return shouldNormalize ? embeddings.map((embedding) => normalizeVector(embedding)) : embeddings;
+  }
+  async embedDocuments(input, options = {}) {
+    return this.embed(input, options);
+  }
+  async embedOne(input, options = {}) {
+    const [embedding] = await this.embed(input, options);
+    if (!embedding) {
+      throw new Error("No embedding returned");
+    }
+    return embedding;
+  }
+  async embedQuery(input, options = {}) {
+    return (await this.embedQueries([input], options))[0];
+  }
+  async embedQueries(input, options = {}) {
+    const snapshot = [...input];
+    return this.enqueue(() => {
+      if (this.isWinzling() && (options.instruction || options.preset)) {
+        throw new Error("Winzling uses unprefixed queries; instructions/presets are not supported");
+      }
+      return this.embedInternal(this.isWinzling() ? snapshot : snapshot.map((query) => formatInstructionQuery(query, options)), options);
+    });
+  }
+  dispose() {
+    if (this.closing) return this.closing;
+    this.closed = true;
+    this.closing = this.queue.then(async () => {
+      const extractor = await this.extractorPromise;
+      await extractor?.dispose?.();
+      this.extractorPromise = null;
+    });
+    return this.closing;
+  }
+  async getExtractor() {
+    if (this.isOpenAICompatibleMode()) {
+      throw new Error(
+        "Local extractor access is unavailable when using an OpenAI-compatible embedding endpoint."
+      );
+    }
+    if (this.extractorPromise !== null) {
+      return this.extractorPromise;
+    }
+    this.extractorPromise = (async () => {
+      const source = this.getLocalModelSource("getExtractor");
+      if (this.isWinzling()) {
+        if (this.dtype !== "q4") throw new Error("Winzling requires dtype: q4 (the model_uint4.onnx artifact)");
+        if (source.kind === "repo" && source.input !== WINZLING_MODEL_ID) throw new Error(`modelProfile "winzling" accepts only ${WINZLING_MODEL_ID} or an HTTP(S) mirror, not ${source.input}`);
+        if (source.kind === "repo" && source.revision !== WINZLING_REVISION) throw new Error("Winzling support is pinned to the audited WINZLING_REVISION");
+        return createWinzlingExtractor({
+          ...this.options.winzling,
+          device: this.options.device ?? this.options.winzling?.device ?? "wasm",
+          cacheDir: this.options.winzling?.cacheDir ?? this.options.cacheDir,
+          allowRemoteModels: this.options.winzling?.allowRemoteModels ?? this.options.allowRemoteModels,
+          ...source.kind === "url" ? { modelBaseUrl: source.input } : {}
+        });
+      }
+      const shouldWarmCacheOnLoad = this.options.warmCacheOnLoad ?? (!this.options.moduleFactory && !this.options.pipelineFactory);
+      if (shouldWarmCacheOnLoad) {
+        await this.prefetchModel(this.model, {
+          revision: source.revision,
+          cacheDir: this.options.cacheDir
+        });
+      }
+      const mod = this.options.moduleFactory ? await this.options.moduleFactory() : await getTransformersModule();
+      configureEnv(mod, this.options, this.runtimeDefaults);
+      mod.env.remoteHost = source.remoteHost;
+      mod.env.remotePathTemplate = source.remotePathTemplate;
+      const pipelineFactory = this.options.pipelineFactory ?? mod.pipeline;
+      return pipelineFactory("feature-extraction", source.modelId, {
+        dtype: this.dtype,
+        ...this.options.device ? { device: this.options.device } : {},
+        ...source.revision ? { revision: source.revision } : {}
+      });
+    })().catch((error) => {
+      this.extractorPromise = null;
+      throw error;
+    });
+    return this.extractorPromise;
+  }
+}
+
+export { BaseEmbeddingRuntime as B };