mikoshi 0.1.0

package/src/mikoshi/retrieval/semantic.js

@@ -0,0 +1,210 @@
+import os from "node:os";
+import path from "node:path";
+
+const DEFAULT_DIM = 384;
+
+function normalizeVector(vec) {
+  let sum = 0;
+  for (let i = 0; i < vec.length; i += 1) sum += vec[i] * vec[i];
+  const norm = Math.sqrt(sum) || 1;
+  for (let i = 0; i < vec.length; i += 1) vec[i] /= norm;
+  return vec;
+}
+
+export function normalizeEmbeddings(matrix, dim) {
+  if (!matrix || !matrix.length || !dim) return matrix;
+  const rows = Math.floor(matrix.length / dim);
+  for (let i = 0; i < rows; i += 1) {
+    const offset = i * dim;
+    let sum = 0;
+    for (let j = 0; j < dim; j += 1) {
+      const v = matrix[offset + j];
+      sum += v * v;
+    }
+    const norm = Math.sqrt(sum) || 1;
+    for (let j = 0; j < dim; j += 1) {
+      matrix[offset + j] /= norm;
+    }
+  }
+  return matrix;
+}
+
+class HashEmbeddingsProvider {
+  constructor(dim = DEFAULT_DIM) {
+    this.dimension = dim;
+  }
+
+  async embedTexts(texts) {
+    const vectors = [];
+    for (const text of texts) {
+      const vec = new Float32Array(this.dimension);
+      const tokens = text.toLowerCase().split(/[^a-z0-9_]+/).filter(Boolean);
+      for (const token of tokens) {
+        let hash = 0;
+        for (let i = 0; i < token.length; i += 1) {
+          hash = (hash * 31 + token.charCodeAt(i)) >>> 0;
+        }
+        const idx = hash % this.dimension;
+        vec[idx] += 1;
+      }
+      normalizeVector(vec);
+      vectors.push(vec);
+    }
+    return vectors;
+  }
+}
+
+class OpenAIEmbeddingsProvider {
+  constructor(config) {
+    this.config = config;
+    this.dimension = null;
+  }
+
+  async embedTexts(texts) {
+    const response = await fetch(`${this.config.embeddings.openai_base_url}/v1/embeddings`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.config.embeddings.openai_key}`,
+      },
+      body: JSON.stringify({
+        model: this.config.embeddings.openai_model,
+        input: texts,
+      }),
+    });
+    if (!response.ok) {
+      const detail = await response.text();
+      throw new Error(`OpenAI embeddings failed: ${detail}`);
+    }
+    const data = await response.json();
+    const vectors = data.data.map((item) => Float32Array.from(item.embedding));
+    if (vectors.length) {
+      this.dimension = vectors[0].length;
+      for (const vec of vectors) normalizeVector(vec);
+    }
+    return vectors;
+  }
+}
+
+class TransformersEmbeddingsProvider {
+  constructor(config) {
+    this.config = config;
+    this.dimension = null;
+    this._pipelinePromise = null;
+  }
+
+  async _loadPipeline() {
+    if (this._pipelinePromise) return this._pipelinePromise;
+    this._pipelinePromise = (async () => {
+      const { pipeline, env } = await import("@xenova/transformers");
+      const cacheDir = path.join(os.homedir(), ".mikoshi", "models");
+      env.cacheDir = cacheDir;
+      env.allowLocalModels = true;
+
+      const offline = isOffline();
+      const timeoutMs = downloadTimeoutMs();
+
+      console.error("🧠 Loading embeddings model…");
+
+      env.allowRemoteModels = false;
+      try {
+        const extractor = await pipeline("feature-extraction", this.config.embeddings.model, {
+          quantized: false,
+        });
+        console.error("✅ Embeddings model loaded");
+        return extractor;
+      } catch (err) {
+        if (offline) {
+          throw new Error(
+            "Model download disabled. Set MIKOSHI_OFFLINE=0 or pre-cache the model."
+          );
+        }
+      }
+
+      console.error("⬇️ Downloading local model (one-time)…");
+      env.allowRemoteModels = true;
+      const extractor = await withTimeout(
+        pipeline("feature-extraction", this.config.embeddings.model, { quantized: false }),
+        timeoutMs
+      );
+      console.error("✅ Model ready");
+      console.error("✅ Embeddings model loaded");
+      return extractor;
+    })();
+    return this._pipelinePromise;
+  }
+
+  async embedTexts(texts) {
+    const extractor = await this._loadPipeline();
+    const vectors = [];
+    for (const text of texts) {
+      const output = await extractor(text, { pooling: "mean", normalize: true });
+      const data = output.data;
+      const vec = Float32Array.from(data);
+      this.dimension = vec.length;
+      vectors.push(vec);
+    }
+    return vectors;
+  }
+}
+
+function isOffline() {
+  const raw = process.env.MIKOSHI_OFFLINE || "";
+  return ["1", "true", "yes", "on"].includes(raw.toLowerCase());
+}
+
+function downloadTimeoutMs() {
+  const raw = process.env.MIKOSHI_MODEL_DOWNLOAD_TIMEOUT_SEC;
+  const seconds = raw ? Number.parseInt(raw, 10) : 300;
+  if (!Number.isFinite(seconds) || seconds <= 0) return 300_000;
+  return seconds * 1000;
+}
+
+function withTimeout(promise, timeoutMs) {
+  let timeoutId;
+  const timeoutPromise = new Promise((_, reject) => {
+    timeoutId = setTimeout(() => {
+      reject(
+        new Error(
+          "Model download timed out. Set MIKOSHI_OFFLINE=1 to skip, or retry on a stable network."
+        )
+      );
+    }, timeoutMs);
+  });
+  return Promise.race([promise, timeoutPromise]).finally(() => clearTimeout(timeoutId));
+}
+
+export function getEmbeddingsProvider(config) {
+  if (process.env.MIKOSHI_TEST_MODE === "1") {
+    return new HashEmbeddingsProvider();
+  }
+  if (config.embeddings.provider === "openai") {
+    return new OpenAIEmbeddingsProvider(config);
+  }
+  return new TransformersEmbeddingsProvider(config);
+}
+
+export class SemanticSearcher {
+  constructor(embeddings, dim, provider) {
+    this.embeddings = embeddings;
+    this.dim = dim;
+    this.provider = provider;
+  }
+
+  async search(query, k) {
+    const vectors = await this.provider.embedTexts([query]);
+    const queryVec = vectors[0];
+    const results = [];
+    const rows = Math.floor(this.embeddings.length / this.dim);
+    for (let i = 0; i < rows; i += 1) {
+      let score = 0;
+      const offset = i * this.dim;
+      for (let j = 0; j < this.dim; j += 1) {
+        score += this.embeddings[offset + j] * queryVec[j];
+      }
+      results.push([i, score]);
+    }
+    results.sort((a, b) => b[1] - a[1]);
+    return results.slice(0, k);
+  }
+}

package/src/mikoshi/utils/timer.js

@@ -0,0 +1,9 @@
+export class Timer {
+  constructor() {
+    this.start = Date.now();
+  }
+
+  get ms() {
+    return Date.now() - this.start;
+  }
+}

package/src/mikoshi/utils/types.js

@@ -0,0 +1,44 @@
+/**
+ * @typedef {Object} Chunk
+ * @property {string} id
+ * @property {string} relpath
+ * @property {number} start_line
+ * @property {number} end_line
+ * @property {string} text
+ * @property {string} file_hash
+ * @property {number | null} vector_idx
+ */
+
+/**
+ * @typedef {Object} IndexMeta
+ * @property {string} repo_id
+ * @property {string} repo_path
+ * @property {string} created_at
+ * @property {string} updated_at
+ * @property {string} embedding_provider
+ * @property {string} model
+ * @property {number} embedding_dim
+ * @property {number} chunk_lines
+ * @property {number} chunk_overlap
+ * @property {number} max_bytes
+ * @property {Object.<string, string>} files
+ * @property {number} chunks
+ */
+
+/**
+ * @typedef {Object} IndexResult
+ * @property {string} repo_id
+ * @property {number} chunks_indexed
+ * @property {number} took_ms
+ */
+
+/**
+ * @typedef {Object} SearchResult
+ * @property {string} relpath
+ * @property {number} start_line
+ * @property {number} end_line
+ * @property {number} score
+ * @property {string} snippet
+ */
+
+export {};