@awareness-sdk/local 0.1.0

package/src/core/config.mjs

@@ -0,0 +1,303 @@
+/**
+ * Config Manager for Awareness Local
+ *
+ * Handles:
+ *   - Device ID generation (unique per machine)
+ *   - .awareness/ directory scaffolding + .gitignore
+ *   - config.json creation, loading, and updating
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import crypto from 'node:crypto';
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const AWARENESS_DIR = '.awareness';
+const CONFIG_FILENAME = 'config.json';
+
+/** Subdirectories to create inside .awareness/ */
+const SUBDIRS = [
+  'memories',
+  'knowledge',
+  'knowledge/decisions',
+  'knowledge/solutions',
+  'knowledge/workflows',
+  'knowledge/insights',
+  'tasks',
+  'tasks/open',
+  'tasks/done',
+];
+
+/** Files/patterns that must NOT be committed to Git */
+const GITIGNORE_CONTENT = `# SQLite index (rebuilt locally on each device)
+index.db
+index.db-journal
+index.db-wal
+
+# Daemon runtime files
+daemon.pid
+daemon.log
+
+# Cloud sync credentials (security-sensitive)
+config.json
+`;
+
+/** Default configuration matching spec section 7.5 */
+const DEFAULT_CONFIG = Object.freeze({
+  version: 1,
+  daemon: {
+    port: 37800,
+    auto_start: true,
+    log_level: 'info',
+  },
+  device: {
+    id: '',   // filled by generateDeviceId()
+    name: '', // filled by hostname
+  },
+  agent: {
+    default_role: 'builder_agent',
+  },
+  extraction: {
+    enabled: true,
+  },
+  embedding: {
+    language: 'english',
+    model_id: null,
+  },
+  cloud: {
+    enabled: false,
+    api_base: 'https://awareness.market/api/v1',
+    api_key: '',
+    memory_id: '',
+    auto_sync: true,
+    sync_interval_min: 5,
+    last_push_at: null,
+    last_pull_at: null,
+    push_cursor: null,
+    pull_cursor: null,
+  },
+  git_sync: {
+    enabled: true,
+    auto_commit: false,
+    branch: null,
+  },
+});
+
+// ---------------------------------------------------------------------------
+// Device ID
+// ---------------------------------------------------------------------------
+
+/**
+ * Generate a deterministic-ish device fingerprint.
+ * Format: "{platform}-{hostname-slug}-{4-hex}"
+ *
+ * The 4-hex suffix is derived from a hash of hostname + homedir + platform
+ * so it is stable across restarts but distinct across machines.
+ *
+ * @returns {string} e.g. "mac-edwins-mbp-a3f2"
+ */
+export function generateDeviceId() {
+  const hostname = os.hostname().toLowerCase();
+  const platform = processPlatformLabel();
+  const slug = slugify(hostname, 20);
+
+  // Deterministic short hash based on multiple machine signals
+  const raw = `${os.hostname()}|${os.homedir()}|${os.platform()}|${os.arch()}`;
+  const hash = crypto.createHash('sha256').update(raw).digest('hex');
+  const suffix = hash.slice(0, 4);
+
+  return `${platform}-${slug}-${suffix}`;
+}
+
+// ---------------------------------------------------------------------------
+// Directory scaffolding
+// ---------------------------------------------------------------------------
+
+/**
+ * Create the full .awareness/ directory tree and write .gitignore.
+ * Safe to call multiple times (idempotent).
+ *
+ * @param {string} projectDir - Absolute path to the project root
+ * @returns {string} Absolute path to the .awareness/ directory
+ */
+export function ensureLocalDirs(projectDir) {
+  const awarenessDir = path.join(projectDir, AWARENESS_DIR);
+
+  // Create root dir
+  fs.mkdirSync(awarenessDir, { recursive: true });
+
+  // Create all subdirectories
+  for (const sub of SUBDIRS) {
+    fs.mkdirSync(path.join(awarenessDir, sub), { recursive: true });
+  }
+
+  // Write .gitignore (overwrite every time to keep it in sync with spec)
+  const gitignorePath = path.join(awarenessDir, '.gitignore');
+  fs.writeFileSync(gitignorePath, GITIGNORE_CONTENT, 'utf-8');
+
+  return awarenessDir;
+}
+
+// ---------------------------------------------------------------------------
+// Config CRUD
+// ---------------------------------------------------------------------------
+
+/**
+ * Return the absolute path to .awareness/config.json
+ *
+ * @param {string} projectDir
+ * @returns {string}
+ */
+export function getConfigPath(projectDir) {
+  return path.join(projectDir, AWARENESS_DIR, CONFIG_FILENAME);
+}
+
+/**
+ * Create config.json with all defaults.  If the file already exists it is
+ * left untouched (use loadLocalConfig + saveCloudConfig to modify).
+ *
+ * @param {string} projectDir
+ * @returns {object} The config object that was written (or already existed)
+ */
+export function initLocalConfig(projectDir) {
+  const configPath = getConfigPath(projectDir);
+
+  // Idempotent — never overwrite an existing config
+  if (fs.existsSync(configPath)) {
+    return loadLocalConfig(projectDir);
+  }
+
+  // Ensure parent dirs exist
+  ensureLocalDirs(projectDir);
+
+  const deviceId = generateDeviceId();
+  const deviceName = os.hostname();
+
+  const config = deepClone(DEFAULT_CONFIG);
+  config.device.id = deviceId;
+  config.device.name = deviceName;
+
+  fs.writeFileSync(configPath, JSON.stringify(config, null, 2), 'utf-8');
+  return config;
+}
+
+/**
+ * Read and return config.json.  Missing keys are filled from DEFAULT_CONFIG
+ * so callers always get a complete shape.
+ *
+ * @param {string} projectDir
+ * @returns {object}
+ */
+export function loadLocalConfig(projectDir) {
+  const configPath = getConfigPath(projectDir);
+
+  if (!fs.existsSync(configPath)) {
+    // Return defaults without writing — caller may want to init explicitly
+    const fallback = deepClone(DEFAULT_CONFIG);
+    fallback.device.id = generateDeviceId();
+    fallback.device.name = os.hostname();
+    return fallback;
+  }
+
+  try {
+    const raw = fs.readFileSync(configPath, 'utf-8');
+    const saved = JSON.parse(raw);
+    // Merge saved over defaults so new keys added in future versions are present
+    return deepMerge(deepClone(DEFAULT_CONFIG), saved);
+  } catch (err) {
+    // Corrupted JSON — return defaults rather than crash
+    const fallback = deepClone(DEFAULT_CONFIG);
+    fallback.device.id = generateDeviceId();
+    fallback.device.name = os.hostname();
+    return fallback;
+  }
+}
+
+/**
+ * Update the cloud section of config.json after device-auth completes.
+ *
+ * @param {string} projectDir
+ * @param {{ apiKey: string, memoryId: string, apiBase?: string }} cloudOpts
+ * @returns {object} The updated full config
+ */
+export function saveCloudConfig(projectDir, { apiKey, memoryId, apiBase }) {
+  const config = loadLocalConfig(projectDir);
+
+  config.cloud.enabled = true;
+  config.cloud.api_key = apiKey;
+  config.cloud.memory_id = memoryId;
+  if (apiBase) {
+    config.cloud.api_base = apiBase;
+  }
+
+  const configPath = getConfigPath(projectDir);
+  // Ensure dirs exist before writing
+  ensureLocalDirs(projectDir);
+  fs.writeFileSync(configPath, JSON.stringify(config, null, 2), 'utf-8');
+
+  return config;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers (internal)
+// ---------------------------------------------------------------------------
+
+/** Map os.platform() to a short human label */
+function processPlatformLabel() {
+  const p = os.platform();
+  if (p === 'darwin') return 'mac';
+  if (p === 'win32') return 'win';
+  if (p === 'linux') return 'linux';
+  return p;
+}
+
+/**
+ * Turn an arbitrary string into a URL/filename-safe slug.
+ * Non-ASCII chars are removed, spaces/underscores become hyphens, consecutive
+ * hyphens are collapsed, and the result is trimmed to maxLen.
+ */
+function slugify(text, maxLen = 50) {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9\s-]/g, '')
+    .replace(/[\s_]+/g, '-')
+    .replace(/-+/g, '-')
+    .replace(/^-|-$/g, '')
+    .slice(0, maxLen);
+}
+
+/** Structured clone polyfill for plain objects */
+function deepClone(obj) {
+  return JSON.parse(JSON.stringify(obj));
+}
+
+/**
+ * Recursively merge `source` into `target`.
+ * - Scalar values in source overwrite target
+ * - Objects are merged recursively
+ * - Arrays in source overwrite target (no concat)
+ */
+function deepMerge(target, source) {
+  for (const key of Object.keys(source)) {
+    const srcVal = source[key];
+    const tgtVal = target[key];
+
+    if (
+      srcVal !== null &&
+      typeof srcVal === 'object' &&
+      !Array.isArray(srcVal) &&
+      tgtVal !== null &&
+      typeof tgtVal === 'object' &&
+      !Array.isArray(tgtVal)
+    ) {
+      target[key] = deepMerge(tgtVal, srcVal);
+    } else {
+      target[key] = srcVal;
+    }
+  }
+  return target;
+}

package/src/core/embedder.mjs

@@ -0,0 +1,239 @@
+/**
+ * Embedder — local embedding module for Awareness Local.
+ *
+ * Uses @huggingface/transformers (ONNX WASM) for purely-in-JS inference.
+ * Falls back gracefully to FTS5-only mode when the dependency is missing.
+ *
+ * Two model options (user-facing names hide actual model identifiers):
+ *   "english"       → Xenova/all-MiniLM-L6-v2       (23 MB, English only)
+ *   "multilingual"  → Xenova/multilingual-e5-small   (118 MB, 100+ languages)
+ *
+ * Both produce 384-dimensional Float32Array vectors.
+ */
+
+// ---------------------------------------------------------------------------
+// Model map
+// ---------------------------------------------------------------------------
+
+export const MODEL_MAP = {
+  english: 'Xenova/all-MiniLM-L6-v2',
+  multilingual: 'Xenova/multilingual-e5-small',
+};
+
+/**
+ * Models whose architecture requires a "query: " / "passage: " prefix.
+ * Currently only the e5 family needs this.
+ */
+const E5_MODELS = new Set([MODEL_MAP.multilingual]);
+
+// ---------------------------------------------------------------------------
+// Pipeline cache (one per language/model)
+// ---------------------------------------------------------------------------
+
+/** @type {Map<string, Promise<any>>} */
+const _pipelineCache = new Map();
+
+/** Whether the HF transformers library is available at all. */
+let _hfAvailable = null; // null = not checked yet, true/false after first probe
+
+/**
+ * Dynamically import @huggingface/transformers.
+ * Returns the module or null if not installed.
+ * @private
+ */
+async function _loadHfModule() {
+  if (_hfAvailable === false) return null;
+  try {
+    const mod = await import('@huggingface/transformers');
+    _hfAvailable = true;
+    return mod;
+  } catch {
+    _hfAvailable = false;
+    console.warn(
+      '[embedder] @huggingface/transformers is not installed. ' +
+        'Embedding-based semantic search is disabled; falling back to FTS5-only mode. ' +
+        'Install it with: npm install @huggingface/transformers'
+    );
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Lazy-load (and cache) the embedding pipeline for the given language.
+ *
+ * @param {string} [language='english'] — 'english' | 'multilingual'
+ * @returns {Promise<Function|null>} — the HF pipeline function, or null if unavailable.
+ */
+export async function getEmbedder(language = 'english') {
+  const modelId = MODEL_MAP[language] || MODEL_MAP.english;
+
+  if (_pipelineCache.has(modelId)) {
+    return _pipelineCache.get(modelId);
+  }
+
+  // Store the promise itself so concurrent callers share the same load.
+  const loadPromise = (async () => {
+    const hf = await _loadHfModule();
+    if (!hf) return null;
+
+    const pipe = await hf.pipeline('feature-extraction', modelId, {
+      dtype: 'q8', // INT8 quantised
+    });
+    return pipe;
+  })();
+
+  _pipelineCache.set(modelId, loadPromise);
+
+  // If the load fails, evict the cache entry so the next call can retry.
+  loadPromise.catch(() => {
+    _pipelineCache.delete(modelId);
+  });
+
+  return loadPromise;
+}
+
+/**
+ * Check whether embedding is available (HF library installed).
+ *
+ * @returns {Promise<boolean>}
+ */
+export async function isEmbeddingAvailable() {
+  if (_hfAvailable !== null) return _hfAvailable;
+  const mod = await _loadHfModule();
+  return mod !== null;
+}
+
+/**
+ * Embed a single text string.
+ *
+ * @param {string} text
+ * @param {string} [type='passage'] — 'query' | 'passage' (affects e5 prefix).
+ * @param {string} [language='english'] — 'english' | 'multilingual'.
+ * @returns {Promise<Float32Array>} — 384-dimensional normalised vector.
+ * @throws {Error} if embedding is unavailable.
+ */
+export async function embed(text, type = 'passage', language = 'english') {
+  const pipe = await getEmbedder(language);
+  if (!pipe) {
+    throw new Error(
+      'Embedding unavailable: @huggingface/transformers is not installed.'
+    );
+  }
+
+  const modelId = MODEL_MAP[language] || MODEL_MAP.english;
+  const input = E5_MODELS.has(modelId) ? `${type}: ${text}` : text;
+
+  const output = await pipe(input, { pooling: 'mean', normalize: true });
+  return new Float32Array(output.data);
+}
+
+/**
+ * Embed multiple texts in a single batch call.
+ *
+ * @param {string[]} texts
+ * @param {string} [type='passage']
+ * @param {string} [language='english']
+ * @returns {Promise<Float32Array[]>}
+ * @throws {Error} if embedding is unavailable.
+ */
+export async function embedBatch(texts, type = 'passage', language = 'english') {
+  if (!texts || texts.length === 0) return [];
+
+  const pipe = await getEmbedder(language);
+  if (!pipe) {
+    throw new Error(
+      'Embedding unavailable: @huggingface/transformers is not installed.'
+    );
+  }
+
+  const modelId = MODEL_MAP[language] || MODEL_MAP.english;
+  const usePrefix = E5_MODELS.has(modelId);
+
+  const inputs = usePrefix ? texts.map((t) => `${type}: ${t}`) : texts;
+
+  const output = await pipe(inputs, { pooling: 'mean', normalize: true });
+
+  // The pipeline returns a nested tensor; output.tolist() gives number[][].
+  // We convert each sub-array to a Float32Array.
+  const dim = 384;
+  const results = [];
+  if (output.data && output.data.length === texts.length * dim) {
+    // Flat buffer — slice into per-text vectors.
+    for (let i = 0; i < texts.length; i++) {
+      results.push(new Float32Array(output.data.slice(i * dim, (i + 1) * dim)));
+    }
+  } else if (typeof output.tolist === 'function') {
+    const nested = output.tolist();
+    for (const row of nested) {
+      results.push(new Float32Array(row));
+    }
+  } else {
+    // Fallback: embed one-by-one.
+    for (const text of texts) {
+      results.push(await embed(text, type, language));
+    }
+  }
+
+  return results;
+}
+
+// ---------------------------------------------------------------------------
+// Vector utilities
+// ---------------------------------------------------------------------------
+
+/**
+ * Cosine similarity between two vectors.
+ *
+ * @param {Float32Array} a
+ * @param {Float32Array} b
+ * @returns {number} — value in [-1, 1].
+ */
+export function cosineSimilarity(a, b) {
+  if (a.length !== b.length) {
+    throw new Error(
+      `Vector dimension mismatch: ${a.length} vs ${b.length}`
+    );
+  }
+
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  if (denom === 0) return 0;
+  return dot / denom;
+}
+
+/**
+ * Convert a Float32Array to a Buffer suitable for SQLite BLOB storage.
+ *
+ * @param {Float32Array} vector
+ * @returns {Buffer}
+ */
+export function vectorToBuffer(vector) {
+  return Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
+}
+
+/**
+ * Convert a Buffer (from SQLite BLOB) back to a Float32Array.
+ *
+ * @param {Buffer} buffer
+ * @returns {Float32Array}
+ */
+export function bufferToVector(buffer) {
+  return new Float32Array(
+    buffer.buffer,
+    buffer.byteOffset,
+    buffer.byteLength / Float32Array.BYTES_PER_ELEMENT
+  );
+}

package/src/core/index.mjs

@@ -0,0 +1,34 @@
+/**
+ * Core module barrel export for Awareness Local
+ *
+ * Re-exports all core modules so consumers can do:
+ *   import { MemoryStore, Indexer, SearchEngine } from './core/index.mjs';
+ */
+
+export {
+  ensureLocalDirs,
+  initLocalConfig,
+  loadLocalConfig,
+  saveCloudConfig,
+  getConfigPath,
+  generateDeviceId,
+} from './config.mjs';
+
+export { MemoryStore } from './memory-store.mjs';
+
+export { Indexer } from './indexer.mjs';
+
+export {
+  getEmbedder,
+  embed,
+  embedBatch,
+  cosineSimilarity,
+  vectorToBuffer,
+  bufferToVector,
+} from './embedder.mjs';
+
+export { SearchEngine } from './search.mjs';
+
+export { KnowledgeExtractor } from './knowledge-extractor.mjs';
+
+export { CloudSync } from './cloud-sync.mjs';