@awareness-sdk/local 0.1.8 → 0.1.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@awareness-sdk/local",
-  "version": "0.1.8",
+  "version": "0.1.10",
   "description": "Local-first AI agent memory system. No account needed.",
   "license": "Apache-2.0",
   "type": "module",

package/src/core/cloud-sync.mjs

@@ -644,6 +644,13 @@ export class CloudSync {
       });
 
     } catch (err) {
+      // SSE is optional (server may not support it yet) — silently fall back
+      if (err.message && err.message.includes('404')) {
+        if (!this._sseRetryCount) {
+          console.log(`${LOG_PREFIX} SSE not available on server — using periodic sync only`);
+        }
+        return; // Don't retry on 404; periodic sync is the fallback
+      }
       console.warn(`${LOG_PREFIX} SSE connection failed:`, err.message);
       this._scheduleSSEReconnect();
     }

package/src/core/indexer.mjs

@@ -7,6 +7,7 @@
 
 import Database from 'better-sqlite3';
 import { createHash } from 'node:crypto';
+import { readFileSync, existsSync } from 'node:fs';
 
 // ---------------------------------------------------------------------------
 // Schema DDL
@@ -31,7 +32,7 @@ CREATE TABLE IF NOT EXISTS memories (
 
 CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
   id UNINDEXED, title, content, tags,
-  tokenize='unicode61 remove_diacritics 2'
+  tokenize='trigram'
 );
 
 CREATE TABLE IF NOT EXISTS knowledge_cards (
@@ -50,7 +51,7 @@ CREATE TABLE IF NOT EXISTS knowledge_cards (
 
 CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_fts USING fts5(
   id UNINDEXED, title, summary, content, tags,
-  tokenize='unicode61 remove_diacritics 2'
+  tokenize='trigram'
 );
 
 CREATE TABLE IF NOT EXISTS tasks (
@@ -155,6 +156,26 @@ export class Indexer {
 
     this.initSchema();
     this._prepareStatements();
+    this._reindexFts();
+    this._checkFtsSyncHealth();
+  }
+
+  /**
+   * If FTS row count is less than memories row count, reindex missing entries.
+   * Handles cases where migration dropped FTS data or records were added without FTS.
+   */
+  _checkFtsSyncHealth() {
+    try {
+      const memCount = this.db.prepare('SELECT count(*) AS c FROM memories').get().c;
+      const ftsCount = this.db.prepare('SELECT count(*) AS c FROM memories_fts').get().c;
+      if (memCount > 0 && ftsCount < memCount) {
+        console.log(`[indexer] FTS out of sync (${ftsCount}/${memCount}) — rebuilding missing entries...`);
+        this._ftsNeedsReindex = true;
+        this._reindexFts();
+      }
+    } catch {
+      // Skip if tables don't exist yet
+    }
   }
 
   // -----------------------------------------------------------------------
@@ -166,9 +187,78 @@ export class Indexer {
    * Safe to call repeatedly — every statement uses IF NOT EXISTS.
    */
   initSchema() {
+    // Migrate FTS5 tables from unicode61 to trigram (CJK support)
+    this._migrateFtsTokenizer();
     this.db.exec(SCHEMA_SQL);
   }
 
+  /**
+   * If existing FTS5 tables use unicode61 tokenizer, recreate them with trigram.
+   * This enables Chinese/Japanese/Korean full-text search.
+   */
+  _migrateFtsTokenizer() {
+    let migrated = false;
+    for (const table of ['memories_fts', 'knowledge_fts']) {
+      try {
+        const row = this.db.prepare(
+          `SELECT sql FROM sqlite_master WHERE type='table' AND name=?`
+        ).get(table);
+        if (row && row.sql && row.sql.includes('unicode61')) {
+          this.db.exec(`DROP TABLE IF EXISTS ${table}`);
+          migrated = true;
+        }
+      } catch {
+        // Table doesn't exist yet — will be created by SCHEMA_SQL
+      }
+    }
+    this._ftsNeedsReindex = migrated;
+  }
+
+  /**
+   * Rebuild FTS5 indexes from source tables after tokenizer migration.
+   * Called after schema init + prepared statements are ready.
+   */
+  _reindexFts() {
+    if (!this._ftsNeedsReindex) return;
+    console.log('[indexer] Rebuilding FTS indexes after tokenizer migration...');
+    // Repopulate memories_fts from memories table + markdown files
+    const memories = this.db.prepare('SELECT id, title, tags, filepath FROM memories').all();
+    for (const m of memories) {
+      try {
+        const content = m.filepath && existsSync(m.filepath)
+          ? readFileSync(m.filepath, 'utf-8')
+          : (m.title || '');
+        this._stmtDeleteFts.run(m.id);
+        this._stmtInsertFts.run({
+          id: m.id,
+          title: m.title || '',
+          content,
+          tags: m.tags || '',
+        });
+      } catch {
+        // Skip files that can't be read
+      }
+    }
+    // Repopulate knowledge_fts
+    const cards = this.db.prepare('SELECT id, title, summary, tags FROM knowledge_cards').all();
+    for (const c of cards) {
+      try {
+        this._stmtDeleteKnowledgeFts.run(c.id);
+        this._stmtInsertKnowledgeFts.run({
+          id: c.id,
+          title: c.title || '',
+          summary: c.summary || '',
+          content: c.summary || '',
+          tags: c.tags || '',
+        });
+      } catch {
+        // Skip
+      }
+    }
+    console.log(`[indexer] FTS reindex done: ${memories.length} memories, ${cards.length} cards`);
+    this._ftsNeedsReindex = false;
+  }
+
   // -----------------------------------------------------------------------
   // Prepared-statement cache (lazy, created once)
   // -----------------------------------------------------------------------

package/src/core/search.mjs

@@ -337,7 +337,7 @@ export class SearchEngine {
         .split(/\s+/)
         .filter((w) => w.length > 0);
       for (const w of words) {
-        // Quote each word for safety (handles special chars)
+        // Quote each word — trigram tokenizer handles CJK natively
         terms.push(`"${w.replace(/"/g, '')}"`);
       }
     }
@@ -510,8 +510,8 @@ export class SearchEngine {
     // Compute cosine similarity for each
     const scored = [];
     for (const item of allEmbeddings) {
-      if (!item.embedding) continue;
-      const similarity = cosineSimilarity(queryVec, item.embedding);
+      if (!item.vector) continue;
+      const similarity = cosineSimilarity(queryVec, item.vector);
       if (similarity > 0.1) {
         scored.push({
           ...item,

package/src/daemon.mjs

@@ -16,7 +16,8 @@
 import http from 'node:http';
 import fs from 'node:fs';
 import path from 'node:path';
-import { fileURLToPath } from 'node:url';
+import { execFile } from 'node:child_process';
+import { fileURLToPath, pathToFileURL } from 'node:url';
 
 import { MemoryStore } from './core/memory-store.mjs';
 import { Indexer } from './core/indexer.mjs';
@@ -886,6 +887,11 @@ export class AwarenessLocalDaemon {
       return await this._apiCloudAuthPoll(req, res);
     }
 
+    // POST /api/v1/cloud/auth/open-browser — open URL in system browser
+    if (route === '/cloud/auth/open-browser' && req.method === 'POST') {
+      return await this._apiCloudAuthOpenBrowser(req, res);
+    }
+
     // GET /api/v1/cloud/memories — list memories (after auth)
     if (route.startsWith('/cloud/memories') && req.method === 'GET') {
       return await this._apiCloudListMemories(req, res, url);
@@ -1137,6 +1143,27 @@ export class AwarenessLocalDaemon {
   // Cloud Auth API (device-auth flow from Dashboard)
   // -----------------------------------------------------------------------
 
+  async _apiCloudAuthOpenBrowser(req, res) {
+    const body = await readBody(req);
+    let params;
+    try { params = JSON.parse(body); } catch { return jsonResponse(res, { error: 'Invalid JSON' }, 400); }
+    const { url: targetUrl } = params;
+    if (!targetUrl || typeof targetUrl !== 'string') {
+      return jsonResponse(res, { error: 'url required' }, 400);
+    }
+    // Only allow opening our own auth URLs
+    if (!targetUrl.startsWith('https://awareness.market/')) {
+      return jsonResponse(res, { error: 'URL not allowed' }, 403);
+    }
+    const cmd = process.platform === 'darwin' ? 'open'
+      : process.platform === 'win32' ? 'start'
+      : 'xdg-open';
+    execFile(cmd, [targetUrl], (err) => {
+      if (err) console.warn('[awareness-local] failed to open browser:', err.message);
+    });
+    return jsonResponse(res, { status: 'ok' });
+  }
+
   async _apiCloudAuthStart(_req, res) {
     const apiBase = this.config?.cloud?.api_base || 'https://awareness.market/api/v1';
     try {
@@ -1178,9 +1205,10 @@ export class AwarenessLocalDaemon {
   }
 
   async _apiCloudListMemories(req, res, url) {
-    // SECURITY C3: Use cloud config API key instead of query param (avoid log/referer leaks)
+    // Accept api_key from query param (during auth flow, before config is saved)
+    // or fall back to saved config (for subsequent calls)
     const config = this._loadConfig();
-    const apiKey = config?.cloud?.api_key;
+    const apiKey = url.searchParams.get('api_key') || config?.cloud?.api_key;
     if (!apiKey) return jsonResponse(res, { error: 'Cloud not configured. Connect via /api/v1/cloud/connect first.' }, 400);
 
     const apiBase = this.config?.cloud?.api_base || 'https://awareness.market/api/v1';
@@ -1370,7 +1398,11 @@ export class AwarenessLocalDaemon {
 
     // Cloud sync (fire-and-forget — don't block the response)
     if (this.cloudSync?.isEnabled()) {
-      this.cloudSync.syncToCloud().catch((err) => {
+      Promise.all([
+        this.cloudSync.syncToCloud(),
+        this.cloudSync.syncInsightsToCloud(),
+        this.cloudSync.syncTasksToCloud(),
+      ]).catch((err) => {
         console.warn('[awareness-local] cloud sync after remember failed:', err.message);
       });
     }
@@ -1784,7 +1816,7 @@ ${item.description || item.title || ''}
       const thisDir = path.dirname(fileURLToPath(import.meta.url));
       const modPath = path.join(thisDir, 'core', 'search.mjs');
       if (fs.existsSync(modPath)) {
-        const mod = await import(modPath);
+        const mod = await import(pathToFileURL(modPath).href);
         const SearchEngine = mod.SearchEngine || mod.default;
         if (SearchEngine) {
           return new SearchEngine(this.indexer, this.memoryStore);
@@ -1802,7 +1834,7 @@ ${item.description || item.title || ''}
       const thisDir = path.dirname(fileURLToPath(import.meta.url));
       const modPath = path.join(thisDir, 'core', 'knowledge-extractor.mjs');
       if (fs.existsSync(modPath)) {
-        const mod = await import(modPath);
+        const mod = await import(pathToFileURL(modPath).href);
         const KnowledgeExtractor = mod.KnowledgeExtractor || mod.default;
         if (KnowledgeExtractor) {
           return new KnowledgeExtractor(this.memoryStore, this.indexer);

package/src/web/index.html

@@ -785,8 +785,14 @@ async function startDeviceAuth() {
     link.href = data.verification_uri + '?code=' + data.user_code;
     link.textContent = data.verification_uri;
 
-    // Open browser automatically
-    window.open(link.href, '_blank');
+    // Open browser via daemon (bypasses popup blockers)
+    api('/cloud/auth/open-browser', {
+      method: 'POST',
+      body: JSON.stringify({ url: link.href })
+    }).catch(function() {
+      // Fallback to window.open if daemon endpoint fails
+      window.open(link.href, '_blank');
+    });
 
     // Poll for authorization
     var result = await api('/cloud/auth/poll', {