eddie-jekyll 0.2.0 → 0.2.4

data/assets/eddie-worker.js

@@ -0,0 +1,226 @@
+// SPDX-License-Identifier: GPL-3.0-only
+
+// Eddie Web Worker
+//
+// Loads WASM module, downloads and caches the ML model in IndexedDB,
+// and handles search queries from the main thread.
+
+"use strict";
+
+// -- Configuration --
+const HF_CDN = "https://huggingface.co";
+const MODEL_FILES = ["config.json", "tokenizer.json", "model.safetensors"];
+const IDB_NAME = "eddie-models";
+const IDB_STORE = "files";
+
+// -- State --
+let baseUrl = "";
+let initialized = false;
+
+// -- Message handler --
+self.onmessage = async function (e) {
+  const msg = e.data;
+
+  if (msg.type === "init") {
+    try {
+      await initialize(msg.indexUrl, msg.baseUrl);
+    } catch (err) {
+      postStatus("error", { error: err.message || String(err) });
+    }
+  } else if (msg.type === "search") {
+    try {
+      if (!initialized) {
+        throw new Error("Engine not initialized");
+      }
+      const mode = msg.mode || "hybrid";
+      const topK = msg.topK || 5;
+      if (typeof wasm_bindgen.search_with_answer !== "function") {
+        throw new Error("WASM does not expose search_with_answer");
+      }
+      const bundle = wasm_bindgen.search_with_answer(
+        msg.query,
+        topK,
+        msg.answerTopK || 5,
+        mode,
+        !!msg.answerMode,
+        msg.qaSubject || ""
+      );
+      self.postMessage({
+        type: "search_result",
+        requestId: msg.requestId,
+        results: bundle.results || [],
+        answer: bundle.answer || null,
+      });
+    } catch (err) {
+      // Keep UX alive on occasional wasm/runtime faults by degrading to keyword lane.
+      try {
+        if (typeof wasm_bindgen.search_with_answer !== "function") {
+          throw err;
+        }
+        const fallback = wasm_bindgen.search_with_answer(
+          msg.query,
+          msg.topK || 5,
+          msg.answerTopK || 5,
+          "keyword",
+          false,
+          msg.qaSubject || ""
+        );
+        self.postMessage({
+          type: "search_result",
+          requestId: msg.requestId,
+          results: fallback.results || [],
+          answer: null,
+          degraded: true,
+          laneError: err.message || String(err),
+        });
+      } catch (fallbackErr) {
+        self.postMessage({
+          type: "error",
+          requestId: msg.requestId,
+          error: fallbackErr.message || fallbackErr.toString(),
+        });
+      }
+    }
+  }
+};
+
+async function initialize(indexUrl, workerBaseUrl) {
+  baseUrl = workerBaseUrl || "";
+
+  // 1. Load WASM glue + instantiate
+  postStatus("loading_wasm");
+  const wasmGlueUrl = resolveUrl("eddie-wasm.js");
+  importScripts(wasmGlueUrl);
+  const wasmBinaryUrl = resolveUrl("eddie.wasm");
+  await wasm_bindgen(wasmBinaryUrl);
+
+  // 2. Fetch index
+  postStatus("loading_index");
+  const indexResponse = await fetch(indexUrl);
+  if (!indexResponse.ok) {
+    throw new Error(`Failed to fetch index: ${indexResponse.status}`);
+  }
+  const indexBytes = new Uint8Array(await indexResponse.arrayBuffer());
+
+  // 3. Parse model ID from index bytes (supports raw .bin and compressed .ed)
+  const modelId = wasm_bindgen.extract_model_id(indexBytes);
+
+  // 4. Fetch model files (with IndexedDB cache)
+  postStatus("checking_cache");
+  const db = await openModelDB();
+
+  const config = await getCachedOrFetch(
+    db,
+    modelId,
+    "config.json",
+    (loaded, total) => postStatus("downloading_model", { progress: loaded / total, file: "config.json" })
+  );
+  const tokenizer = await getCachedOrFetch(
+    db,
+    modelId,
+    "tokenizer.json",
+    (loaded, total) => postStatus("downloading_model", { progress: loaded / total, file: "tokenizer.json" })
+  );
+  const weights = await getCachedOrFetch(
+    db,
+    modelId,
+    "model.safetensors",
+    (loaded, total) => postStatus("downloading_model", { progress: loaded / total, file: "model.safetensors" })
+  );
+
+  // 5. Initialize WASM engine
+  postStatus("initializing");
+  wasm_bindgen.init_engine(
+    new Uint8Array(config),
+    new Uint8Array(tokenizer),
+    new Uint8Array(weights),
+    indexBytes
+  );
+
+  initialized = true;
+  postStatus("ready");
+}
+
+// -- IndexedDB helpers --
+function openModelDB() {
+  return new Promise((resolve, reject) => {
+    const req = indexedDB.open(IDB_NAME, 1);
+    req.onupgradeneeded = () => {
+      req.result.createObjectStore(IDB_STORE);
+    };
+    req.onsuccess = () => resolve(req.result);
+    req.onerror = () => reject(req.error);
+  });
+}
+
+function idbGet(db, key) {
+  return new Promise((resolve, reject) => {
+    const tx = db.transaction(IDB_STORE, "readonly");
+    const req = tx.objectStore(IDB_STORE).get(key);
+    req.onsuccess = () => resolve(req.result);
+    req.onerror = () => reject(req.error);
+  });
+}
+
+function idbPut(db, key, value) {
+  return new Promise((resolve, reject) => {
+    const tx = db.transaction(IDB_STORE, "readwrite");
+    const req = tx.objectStore(IDB_STORE).put(value, key);
+    req.onsuccess = () => resolve();
+    req.onerror = () => reject(req.error);
+  });
+}
+
+async function getCachedOrFetch(db, modelId, filename, onProgress) {
+  const key = `${modelId}/${filename}`;
+  const cached = await idbGet(db, key);
+  if (cached) {
+    return cached;
+  }
+
+  const url = `${HF_CDN}/${modelId}/resolve/main/${filename}`;
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to download ${filename}: ${response.status}`);
+  }
+
+  const contentLength = parseInt(response.headers.get("Content-Length") || "0", 10);
+  const reader = response.body.getReader();
+  const chunks = [];
+  let loaded = 0;
+
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+    loaded += value.length;
+    if (contentLength > 0) {
+      onProgress(loaded, contentLength);
+    }
+  }
+
+  // Concatenate chunks into single ArrayBuffer
+  const total = chunks.reduce((sum, c) => sum + c.length, 0);
+  const result = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) {
+    result.set(chunk, offset);
+    offset += chunk.length;
+  }
+
+  const buffer = result.buffer;
+  await idbPut(db, key, buffer);
+  return buffer;
+}
+
+// -- Utilities --
+function resolveUrl(filename) {
+  if (baseUrl) {
+    return baseUrl.replace(/\/$/, "") + "/" + filename;
+  }
+  return filename;
+}
+
+function postStatus(state, extra) {
+  self.postMessage({ type: "status", state, ...extra });
+}

data/exe/eddie-jekyll-install

@@ -11,6 +11,12 @@ end
 
 root = Pathname.new(__dir__).join("..").expand_path
 script = root.join("scripts", "install.sh")
-asset_root = ARGV[1] || "/repo/dist"
+env = {
+  "EDDIE_PACKAGE_ROOT" => root.to_s
+}
 
-exec("bash", script.to_s, File.expand_path(site_dir), File.expand_path(asset_root))
+if ARGV[1]
+  exec(env, "bash", script.to_s, File.expand_path(site_dir), File.expand_path(ARGV[1]))
+else
+  exec(env, "bash", script.to_s, File.expand_path(site_dir))
+end

data/lib/eddie/jekyll.rb

@@ -2,6 +2,6 @@
 
 module Eddie
   module Jekyll
-    VERSION = "0.2.0"
+    VERSION = "0.2.4"
   end
 end

data/scripts/install.sh

@@ -2,7 +2,19 @@
 set -euo pipefail
 
 SITE_DIR="${1:?usage: install.sh <jekyll-site-dir>}"
-ASSET_ROOT="${2:-/repo/dist}"
+ASSET_ROOT="${2:-}"
+PACKAGE_ROOT="${EDDIE_PACKAGE_ROOT:-}"
+ASSETS=(eddie-widget.js eddie-worker.js eddie-wasm.js eddie.wasm)
+
+if [[ -z "$ASSET_ROOT" && -n "$PACKAGE_ROOT" ]]; then
+  ASSET_ROOT="$PACKAGE_ROOT/assets"
+fi
+
+if [[ -z "$ASSET_ROOT" ]]; then
+  echo "No asset root provided and no packaged assets found." >&2
+  echo "Pass an explicit asset-root or set EDDIE_PACKAGE_ROOT." >&2
+  exit 1
+fi
 
 require_asset() {
   local asset_name="$1"
@@ -13,7 +25,7 @@ require_asset() {
   fi
 }
 
-for asset in eddie-widget.js eddie-worker.js eddie-wasm.js eddie.wasm; do
+for asset in "${ASSETS[@]}"; do
   require_asset "$asset"
 done
 

metadata

@@ -1,14 +1,34 @@
 --- !ruby/object:Gem::Specification
 name: eddie-jekyll
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.4
 platform: ruby
 authors:
 - Jason Grey
 bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
-dependencies: []
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: jt55401-eddie-cli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.4
 description: Provides a CLI helper that runs Eddie's Jekyll installer script.
 executables:
 - eddie-jekyll-install
@@ -17,6 +37,10 @@ extra_rdoc_files: []
 files:
 - LICENSE.txt
 - README.md
+- assets/eddie-wasm.js
+- assets/eddie-widget.js
+- assets/eddie-worker.js
+- assets/eddie.wasm
 - exe/eddie-jekyll-install
 - lib/eddie/jekyll.rb
 - scripts/install.sh