ocean-brain 0.8.2 → 0.10.0

package/server/dist/features/search/sqlite-vector-index.js

@@ -0,0 +1,636 @@
+import { mkdir } from "node:fs/promises";
+import path from "node:path";
+import { getLoadablePath } from "sqlite-vec";
+import sqlite3 from "sqlite3";
+const SEMANTIC_INDEX_SCHEMA_VERSION = 2;
+const SEARCH_DATABASE_SCHEMA_VERSION = 1;
+const SEARCH_DATABASE_SCHEMA_SQL = `
+    CREATE TABLE IF NOT EXISTS search_index_state (
+        id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1),
+        schema_version INTEGER NOT NULL,
+        profile_json TEXT NOT NULL,
+        note_count INTEGER NOT NULL,
+        chunk_count INTEGER NOT NULL,
+        indexed_at TEXT NOT NULL
+    );
+    CREATE TABLE IF NOT EXISTS search_chunks (
+        id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+        note_id INTEGER NOT NULL,
+        chunk_index INTEGER NOT NULL,
+        source_hash TEXT NOT NULL,
+        text TEXT NOT NULL,
+        embedding BLOB NOT NULL,
+        UNIQUE (note_id, chunk_index)
+    );
+    CREATE INDEX IF NOT EXISTS search_chunks_note_id_idx ON search_chunks(note_id);
+    CREATE TABLE IF NOT EXISTS search_note_sync_queue (
+        note_id INTEGER NOT NULL PRIMARY KEY,
+        version INTEGER NOT NULL,
+        first_queued_at INTEGER NOT NULL,
+        last_queued_at INTEGER NOT NULL,
+        attempt_count INTEGER NOT NULL,
+        next_attempt_at INTEGER NOT NULL,
+        last_error TEXT
+    );
+    CREATE INDEX IF NOT EXISTS search_note_sync_queue_ready_idx
+        ON search_note_sync_queue(next_attempt_at, first_queued_at);
+    CREATE TABLE IF NOT EXISTS search_note_sync_state (
+        id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1),
+        last_synced_at TEXT,
+        last_reconciled_at TEXT
+    );
+`;
+const RESET_SEARCH_DATABASE_SCHEMA_SQL = `
+    DROP TABLE IF EXISTS search_note_sync_queue;
+    DROP TABLE IF EXISTS search_note_sync_state;
+    DROP TABLE IF EXISTS search_chunks;
+    DROP TABLE IF EXISTS search_index_state;
+`;
+const openDatabase = async (filePath) => {
+  if (filePath !== ":memory:") {
+    await mkdir(path.dirname(filePath), { recursive: true });
+  }
+  const database = await new Promise((resolve, reject) => {
+    const db = new sqlite3.Database(filePath, (error) => {
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve(db);
+    });
+  });
+  await new Promise((resolve, reject) => {
+    database.loadExtension(getLoadablePath(), (error) => {
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve();
+    });
+  });
+  return database;
+};
+const run = (database, sql, parameters = []) => {
+  return new Promise((resolve, reject) => {
+    database.run(sql, parameters, function onRun(error) {
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve(this);
+    });
+  });
+};
+const get = (database, sql, parameters = []) => {
+  return new Promise((resolve, reject) => {
+    database.get(sql, parameters, (error, row) => {
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve(row);
+    });
+  });
+};
+const all = (database, sql, parameters = []) => {
+  return new Promise((resolve, reject) => {
+    database.all(sql, parameters, (error, rows) => {
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve(rows);
+    });
+  });
+};
+const exec = (database, sql) => {
+  return new Promise((resolve, reject) => {
+    database.exec(sql, (error) => {
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve();
+    });
+  });
+};
+const initializeSearchDatabase = async (database) => {
+  await exec(
+    database,
+    `
+            PRAGMA journal_mode = WAL;
+            PRAGMA synchronous = NORMAL;
+        `
+  );
+  const schemaVersion = await get(
+    database,
+    "SELECT user_version AS userVersion FROM pragma_user_version"
+  );
+  const storedVersion = schemaVersion?.userVersion ?? 0;
+  if (storedVersion !== 0 && storedVersion !== SEARCH_DATABASE_SCHEMA_VERSION) {
+    await exec(database, RESET_SEARCH_DATABASE_SCHEMA_SQL);
+  }
+  await exec(database, SEARCH_DATABASE_SCHEMA_SQL);
+  await exec(database, `PRAGMA user_version = ${SEARCH_DATABASE_SCHEMA_VERSION};`);
+};
+const close = (database) => {
+  return new Promise((resolve, reject) => {
+    database.close((error) => {
+      if (error) {
+        reject(error);
+        return;
+      }
+      resolve();
+    });
+  });
+};
+const serializeEmbedding = (embedding) => {
+  return Buffer.from(Float32Array.from(embedding).buffer);
+};
+const isSemanticIndexProfile = (value) => {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+  const profile = value;
+  return typeof profile.model === "string" && typeof profile.baseUrl === "string" && Number.isInteger(profile.dimensions) && Number(profile.dimensions) > 0 && typeof profile.queryInstruction === "string" && Number.isInteger(profile.textSchemaVersion);
+};
+const parseProfile = (profileJson) => {
+  try {
+    const profile = JSON.parse(profileJson);
+    return isSemanticIndexProfile(profile) ? profile : null;
+  } catch {
+    return null;
+  }
+};
+const validateIndexInput = (profile, chunks) => {
+  if (!profile.model.trim()) {
+    throw new Error("Semantic index profile model is required.");
+  }
+  if (!profile.baseUrl.trim()) {
+    throw new Error("Semantic index profile API URL is required.");
+  }
+  if (!Number.isInteger(profile.dimensions) || profile.dimensions <= 0) {
+    throw new Error("Semantic index dimensions must be a positive integer.");
+  }
+  for (const chunk of chunks) {
+    if (chunk.embedding.length !== profile.dimensions) {
+      throw new Error(
+        `Embedding dimensions for note ${chunk.noteId} chunk ${chunk.chunkIndex} do not match the index profile.`
+      );
+    }
+    if (chunk.embedding.some((value) => !Number.isFinite(value))) {
+      throw new Error(`Embedding for note ${chunk.noteId} chunk ${chunk.chunkIndex} contains an invalid value.`);
+    }
+  }
+};
+class SqliteSemanticVectorIndex {
+  constructor(filePath) {
+    this.filePath = filePath;
+  }
+  filePath;
+  databasePromise = null;
+  operationTail = Promise.resolve();
+  getDatabase() {
+    if (!this.databasePromise) {
+      this.databasePromise = openDatabase(this.filePath).then(async (database) => {
+        await initializeSearchDatabase(database);
+        return database;
+      });
+    }
+    return this.databasePromise;
+  }
+  runExclusive(operation) {
+    const result = this.operationTail.then(operation, operation);
+    this.operationTail = result.then(
+      () => void 0,
+      () => void 0
+    );
+    return result;
+  }
+  async replaceAll(profile, chunks) {
+    validateIndexInput(profile, chunks);
+    return this.runExclusive(async () => {
+      const database = await this.getDatabase();
+      const indexedAt = (/* @__PURE__ */ new Date()).toISOString();
+      const noteCount = new Set(chunks.map((chunk) => chunk.noteId)).size;
+      await exec(database, "BEGIN IMMEDIATE;");
+      try {
+        await run(database, "DELETE FROM search_chunks");
+        await run(database, "DELETE FROM search_index_state");
+        for (const chunk of chunks) {
+          await run(
+            database,
+            `
+                            INSERT INTO search_chunks (
+                                note_id,
+                                chunk_index,
+                                source_hash,
+                                text,
+                                embedding
+                            ) VALUES (?, ?, ?, ?, ?)
+                        `,
+            [
+              chunk.noteId,
+              chunk.chunkIndex,
+              chunk.sourceHash,
+              chunk.text,
+              serializeEmbedding(chunk.embedding)
+            ]
+          );
+        }
+        await run(
+          database,
+          `
+                        INSERT INTO search_index_state (
+                            id,
+                            schema_version,
+                            profile_json,
+                            note_count,
+                            chunk_count,
+                            indexed_at
+                        ) VALUES (1, ?, ?, ?, ?, ?)
+                    `,
+          [SEMANTIC_INDEX_SCHEMA_VERSION, JSON.stringify(profile), noteCount, chunks.length, indexedAt]
+        );
+        await exec(database, "COMMIT;");
+      } catch (error) {
+        await exec(database, "ROLLBACK;");
+        throw error;
+      }
+      return this.getStatusFromDatabase(database);
+    });
+  }
+  async getStatusFromDatabase(database) {
+    const state = await get(
+      database,
+      `
+                SELECT
+                    profile_json AS profileJson,
+                    note_count AS noteCount,
+                    chunk_count AS chunkCount,
+                    indexed_at AS indexedAt
+                FROM search_index_state
+                WHERE id = 1 AND schema_version = ?
+            `,
+      [SEMANTIC_INDEX_SCHEMA_VERSION]
+    );
+    const profile = state ? parseProfile(state.profileJson) : null;
+    return {
+      ready: Boolean(state && profile && state.chunkCount > 0),
+      profile,
+      noteCount: state?.noteCount ?? 0,
+      chunkCount: state?.chunkCount ?? 0,
+      indexedAt: state?.indexedAt ?? null
+    };
+  }
+  async refreshIndexCounts(database) {
+    const counts = await get(
+      database,
+      `
+                SELECT
+                    COUNT(DISTINCT note_id) AS noteCount,
+                    COUNT(*) AS chunkCount
+                FROM search_chunks
+            `
+    );
+    await run(
+      database,
+      `
+                UPDATE search_index_state
+                SET note_count = ?, chunk_count = ?, indexed_at = ?
+                WHERE id = 1 AND schema_version = ?
+            `,
+      [counts?.noteCount ?? 0, counts?.chunkCount ?? 0, (/* @__PURE__ */ new Date()).toISOString(), SEMANTIC_INDEX_SCHEMA_VERSION]
+    );
+  }
+  async getStatus() {
+    return this.runExclusive(async () => this.getStatusFromDatabase(await this.getDatabase()));
+  }
+  async getNoteSourceHash(noteId) {
+    return this.runExclusive(async () => {
+      const row = await get(
+        await this.getDatabase(),
+        "SELECT source_hash AS sourceHash FROM search_chunks WHERE note_id = ? LIMIT 1",
+        [noteId]
+      );
+      return row?.sourceHash ?? null;
+    });
+  }
+  async getAllNoteSourceHashes() {
+    return this.runExclusive(async () => {
+      const rows = await all(
+        await this.getDatabase(),
+        `
+                    SELECT note_id AS noteId, source_hash AS sourceHash
+                    FROM search_chunks
+                    GROUP BY note_id, source_hash
+                    ORDER BY note_id ASC
+                `
+      );
+      return new Map(rows.map((row) => [row.noteId, row.sourceHash]));
+    });
+  }
+  async enqueueNoteSync(noteId, queuedAt) {
+    if (!Number.isInteger(noteId) || noteId <= 0) {
+      throw new Error("A positive note ID is required to queue semantic search synchronization.");
+    }
+    if (!Number.isFinite(queuedAt) || queuedAt < 0) {
+      throw new Error("A valid queue timestamp is required for semantic search synchronization.");
+    }
+    return this.runExclusive(async () => {
+      await run(
+        await this.getDatabase(),
+        `
+                    INSERT INTO search_note_sync_queue (
+                        note_id,
+                        version,
+                        first_queued_at,
+                        last_queued_at,
+                        attempt_count,
+                        next_attempt_at,
+                        last_error
+                    ) VALUES (?, 1, ?, ?, 0, ?, NULL)
+                    ON CONFLICT(note_id) DO UPDATE SET
+                        version = search_note_sync_queue.version + 1,
+                        last_queued_at = excluded.last_queued_at,
+                        attempt_count = 0,
+                        next_attempt_at = excluded.next_attempt_at,
+                        last_error = NULL
+                `,
+        [noteId, queuedAt, queuedAt, queuedAt]
+      );
+    });
+  }
+  async listPendingNoteSyncs({
+    now,
+    quietPeriodMs,
+    maxWaitMs,
+    limit,
+    force = false
+  }) {
+    if (![now, quietPeriodMs, maxWaitMs].every((value) => Number.isFinite(value) && value >= 0)) {
+      throw new Error("Semantic search synchronization timing values must be non-negative numbers.");
+    }
+    if (!Number.isInteger(limit) || limit <= 0) {
+      throw new Error("Semantic search synchronization batch limit must be a positive integer.");
+    }
+    return this.runExclusive(async () => {
+      return all(
+        await this.getDatabase(),
+        `
+                    SELECT
+                        note_id AS noteId,
+                        version,
+                        first_queued_at AS firstQueuedAt,
+                        last_queued_at AS lastQueuedAt,
+                        attempt_count AS attemptCount
+                    FROM search_note_sync_queue
+                    WHERE
+                        ? = 1
+                        OR (
+                            next_attempt_at <= ?
+                            AND (
+                                last_queued_at <= ?
+                                OR first_queued_at <= ?
+                            )
+                        )
+                    ORDER BY first_queued_at ASC, note_id ASC
+                    LIMIT ?
+                `,
+        [force ? 1 : 0, now, now - quietPeriodMs, now - maxWaitMs, limit]
+      );
+    });
+  }
+  async completeNoteSyncs(entries, completedAt) {
+    return this.runExclusive(async () => {
+      const database = await this.getDatabase();
+      await exec(database, "BEGIN IMMEDIATE;");
+      try {
+        for (const entry of entries) {
+          await run(database, "DELETE FROM search_note_sync_queue WHERE note_id = ? AND version = ?", [
+            entry.noteId,
+            entry.version
+          ]);
+        }
+        await run(
+          database,
+          `
+                        INSERT INTO search_note_sync_state (id, last_synced_at, last_reconciled_at)
+                        VALUES (1, ?, NULL)
+                        ON CONFLICT(id) DO UPDATE SET last_synced_at = excluded.last_synced_at
+                    `,
+          [new Date(completedAt).toISOString()]
+        );
+        await exec(database, "COMMIT;");
+      } catch (error) {
+        await exec(database, "ROLLBACK;");
+        throw error;
+      }
+    });
+  }
+  async failNoteSyncs(entries, error, retryAt) {
+    return this.runExclusive(async () => {
+      const database = await this.getDatabase();
+      await exec(database, "BEGIN IMMEDIATE;");
+      try {
+        for (const entry of entries) {
+          await run(
+            database,
+            `
+                            UPDATE search_note_sync_queue
+                            SET attempt_count = attempt_count + 1,
+                                next_attempt_at = ?,
+                                last_error = ?
+                            WHERE note_id = ? AND version = ?
+                        `,
+            [retryAt, error, entry.noteId, entry.version]
+          );
+        }
+        await exec(database, "COMMIT;");
+      } catch (transactionError) {
+        await exec(database, "ROLLBACK;");
+        throw transactionError;
+      }
+    });
+  }
+  async recordNoteSyncSuccess(completedAt) {
+    return this.completeNoteSyncs([], completedAt);
+  }
+  async recordNoteSyncReconciliation(reconciledAt) {
+    return this.runExclusive(async () => {
+      await run(
+        await this.getDatabase(),
+        `
+                    INSERT INTO search_note_sync_state (id, last_synced_at, last_reconciled_at)
+                    VALUES (1, NULL, ?)
+                    ON CONFLICT(id) DO UPDATE SET last_reconciled_at = excluded.last_reconciled_at
+                `,
+        [new Date(reconciledAt).toISOString()]
+      );
+    });
+  }
+  async getNoteSyncQueueStatus() {
+    return this.runExclusive(async () => {
+      const database = await this.getDatabase();
+      const [queue, state, latestError] = await Promise.all([
+        get(
+          database,
+          `
+                        SELECT
+                            COUNT(*) AS pendingNoteCount,
+                            MIN(first_queued_at) AS oldestQueuedAt
+                        FROM search_note_sync_queue
+                    `
+        ),
+        get(
+          database,
+          `
+                        SELECT
+                            last_synced_at AS lastSyncedAt,
+                            last_reconciled_at AS lastReconciledAt
+                        FROM search_note_sync_state
+                        WHERE id = 1
+                    `
+        ),
+        get(
+          database,
+          `
+                        SELECT last_error AS error
+                        FROM search_note_sync_queue
+                        WHERE last_error IS NOT NULL
+                        ORDER BY next_attempt_at DESC, note_id ASC
+                        LIMIT 1
+                    `
+        )
+      ]);
+      return {
+        pendingNoteCount: queue?.pendingNoteCount ?? 0,
+        oldestQueuedAt: queue?.oldestQueuedAt === null || queue?.oldestQueuedAt === void 0 ? null : new Date(queue.oldestQueuedAt).toISOString(),
+        lastSyncedAt: state?.lastSyncedAt ?? null,
+        lastReconciledAt: state?.lastReconciledAt ?? null,
+        error: latestError?.error ?? null
+      };
+    });
+  }
+  async replaceNote(noteId, chunks) {
+    return this.runExclusive(async () => {
+      const database = await this.getDatabase();
+      const status = await this.getStatusFromDatabase(database);
+      if (!status.ready || !status.profile) {
+        throw new Error("Build the semantic index before updating an individual note.");
+      }
+      if (chunks.some((chunk) => chunk.noteId !== noteId)) {
+        throw new Error("Every replacement chunk must belong to the requested note.");
+      }
+      validateIndexInput(status.profile, chunks);
+      await exec(database, "BEGIN IMMEDIATE;");
+      try {
+        await run(database, "DELETE FROM search_chunks WHERE note_id = ?", [noteId]);
+        for (const chunk of chunks) {
+          await run(
+            database,
+            `
+                            INSERT INTO search_chunks (
+                                note_id,
+                                chunk_index,
+                                source_hash,
+                                text,
+                                embedding
+                            ) VALUES (?, ?, ?, ?, ?)
+                        `,
+            [
+              chunk.noteId,
+              chunk.chunkIndex,
+              chunk.sourceHash,
+              chunk.text,
+              serializeEmbedding(chunk.embedding)
+            ]
+          );
+        }
+        await this.refreshIndexCounts(database);
+        await exec(database, "COMMIT;");
+      } catch (error) {
+        await exec(database, "ROLLBACK;");
+        throw error;
+      }
+      return this.getStatusFromDatabase(database);
+    });
+  }
+  async removeNote(noteId) {
+    return this.runExclusive(async () => {
+      const database = await this.getDatabase();
+      await exec(database, "BEGIN IMMEDIATE;");
+      try {
+        await run(database, "DELETE FROM search_chunks WHERE note_id = ?", [noteId]);
+        await this.refreshIndexCounts(database);
+        await exec(database, "COMMIT;");
+      } catch (error) {
+        await exec(database, "ROLLBACK;");
+        throw error;
+      }
+      return this.getStatusFromDatabase(database);
+    });
+  }
+  async search(queryEmbedding, limit) {
+    if (!Number.isInteger(limit) || limit <= 0) {
+      return [];
+    }
+    return this.runExclusive(async () => {
+      const database = await this.getDatabase();
+      const status = await this.getStatusFromDatabase(database);
+      if (!status.ready || !status.profile) {
+        return [];
+      }
+      if (queryEmbedding.length !== status.profile.dimensions) {
+        throw new Error(
+          `Query embedding has ${queryEmbedding.length} dimensions but the index requires ${status.profile.dimensions}.`
+        );
+      }
+      if (queryEmbedding.some((value) => !Number.isFinite(value))) {
+        throw new Error("Query embedding contains an invalid value.");
+      }
+      const rows = await all(
+        database,
+        `
+                    SELECT
+                        note_id AS noteId,
+                        MIN(vec_distance_cosine(embedding, ?)) AS distance
+                    FROM search_chunks
+                    GROUP BY note_id
+                    ORDER BY distance ASC, note_id ASC
+                    LIMIT ?
+                `,
+        [serializeEmbedding(queryEmbedding), limit]
+      );
+      return rows.map((row) => ({ noteId: row.noteId, distance: row.distance }));
+    });
+  }
+  async clear() {
+    return this.runExclusive(async () => {
+      const database = await this.getDatabase();
+      await exec(database, "BEGIN IMMEDIATE;");
+      try {
+        await run(database, "DELETE FROM search_chunks");
+        await run(database, "DELETE FROM search_index_state");
+        await exec(database, "COMMIT;");
+      } catch (error) {
+        await exec(database, "ROLLBACK;");
+        throw error;
+      }
+    });
+  }
+  async close() {
+    await this.operationTail;
+    if (!this.databasePromise) {
+      return;
+    }
+    const database = await this.databasePromise;
+    this.databasePromise = null;
+    await close(database);
+  }
+}
+export {
+  SEARCH_DATABASE_SCHEMA_VERSION,
+  SEMANTIC_INDEX_SCHEMA_VERSION,
+  SqliteSemanticVectorIndex
+};

package/server/dist/features/search/url-normalization.js

@@ -0,0 +1,10 @@
+const stripTrailingSlashes = (value) => {
+  let end = value.length;
+  while (end > 0 && value.charCodeAt(end - 1) === 47) {
+    end -= 1;
+  }
+  return end === value.length ? value : value.slice(0, end);
+};
+export {
+  stripTrailingSlashes
+};

package/server/dist/features/tag/services/organization.js

@@ -10,6 +10,9 @@ const normalizeTagName = (name) => {
   if (!trimmedName) {
     throw new InvalidTagNameError("A tag name is required.");
   }
+  if (trimmedName.startsWith("#")) {
+    throw new InvalidTagNameError("Tag names must use @, not #. Example: @project.");
+  }
   const normalizedName = trimmedName.startsWith("@") ? trimmedName : `@${trimmedName}`;
   if (normalizedName === "@" || /\s/.test(normalizedName.slice(1))) {
     throw new InvalidTagNameError("Tag names must be a single token like @project.");

package/server/dist/features/view/services/workspace.js

@@ -36,7 +36,7 @@ const normalizeTagName = (value) => {
     return trimmedValue;
   }
   if (trimmedValue.startsWith("#")) {
-    return `@${trimmedValue.slice(1)}`;
+    throw new Error("View tag names must use @, not #.");
   }
   return `@${trimmedValue}`;
 };

package/server/dist/main.js

@@ -1,4 +1,5 @@
 import { createApp } from "./app.js";
+import { getDefaultSemanticSearchManager } from "./features/search/search-manager.js";
 import { logAuthConfig, resolveAuthConfig } from "./modules/auth-mode.js";
 import { startDataMaintenanceScheduler } from "./modules/data-maintenance.js";
 const PORT = Number(process.env.PORT || 6683);
@@ -7,6 +8,7 @@ try {
   const authConfig = resolveAuthConfig(process.env);
   logAuthConfig(authConfig);
   const app = createApp(authConfig);
+  getDefaultSemanticSearchManager();
   app.listen(PORT, HOST, () => {
     process.stdout.write(`http server listen on ${HOST}:${PORT} (auth: ${authConfig.mode})
 `);