@vox-ai-app/storage 1.0.1 → 1.0.3

package/README.md

@@ -0,0 +1,109 @@
+# @vox-ai-app/storage
+
+Local persistence for Vox: conversations, messages, tasks, settings, tool registry, MCP servers, schedules, secrets, patterns, and vector embeddings. Built on SQLite via `better-sqlite3` with WAL mode and automatic migrations.
+
+## Install
+
+```sh
+npm install @vox-ai-app/storage
+```
+
+## Exports
+
+| Export                             | Contents                        |
+| ---------------------------------- | ------------------------------- |
+| `@vox-ai-app/storage`              | All exports                     |
+| `@vox-ai-app/storage/db`           | Database lifecycle (open/close) |
+| `@vox-ai-app/storage/messages`     | Conversations and messages      |
+| `@vox-ai-app/storage/tasks`        | Task and task activity storage  |
+| `@vox-ai-app/storage/tools`        | Custom tool definitions         |
+| `@vox-ai-app/storage/settings`     | Key-value settings persistence  |
+| `@vox-ai-app/storage/mcp-servers`  | MCP server configurations       |
+| `@vox-ai-app/storage/schedules`    | Scheduled job persistence       |
+| `@vox-ai-app/storage/tool-secrets` | Encrypted tool secret storage   |
+| `@vox-ai-app/storage/patterns`     | Conversation pattern storage    |
+| `@vox-ai-app/storage/vectors`      | Vector embedding storage        |
+
+## Database
+
+```js
+import { openDb, closeDb } from '@vox-ai-app/storage/db'
+
+const db = openDb('/path/to/storage.db')
+closeDb('/path/to/storage.db')
+```
+
+The database uses WAL journal mode and foreign keys. Schema is managed via migrations in `src/migrations/`.
+
+## Messages
+
+```js
+import {
+  ensureConversation,
+  appendMessage,
+  getMessages,
+  getMessagesBeforeId,
+  clearMessages,
+  saveSummaryCheckpoint,
+  loadSummaryCheckpoint
+} from '@vox-ai-app/storage/messages'
+
+ensureConversation(db, 'main')
+appendMessage(db, 'user', 'Hello', 'main')
+appendMessage(db, 'assistant', 'Hi there!', 'main')
+
+const messages = getMessages(db, 'main', 50)
+const older = getMessagesBeforeId(db, messages[0].id, 'main', 20)
+
+saveSummaryCheckpoint(db, 'summary text', 42, 'main')
+const { summary, checkpointId } = loadSummaryCheckpoint(db, 'main')
+
+clearMessages(db, 'main')
+```
+
+## Tasks
+
+```js
+import {
+  upsertTask,
+  getTask,
+  loadTasks,
+  appendTaskActivity,
+  loadTaskActivity
+} from '@vox-ai-app/storage/tasks'
+
+upsertTask(db, {
+  taskId: 'abc-123',
+  instructions: 'Summarize the document',
+  status: 'running'
+})
+
+const task = getTask(db, 'abc-123')
+const allTasks = loadTasks(db)
+
+appendTaskActivity(db, {
+  id: 'act-1',
+  taskId: 'abc-123',
+  type: 'tool_call',
+  name: 'read_local_file',
+  timestamp: new Date().toISOString(),
+  data: { path: '~/doc.md' }
+})
+
+const activity = loadTaskActivity(db, 'abc-123')
+```
+
+## Settings
+
+```js
+import { getSetting, setSetting, getAllSettings, deleteSetting } from '@vox-ai-app/storage/settings'
+
+setSetting(db, 'theme', 'dark')
+const theme = getSetting(db, 'theme')
+const all = getAllSettings(db)
+deleteSetting(db, 'theme')
+```
+
+## License
+
+MIT

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vox-ai-app/storage",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "type": "module",
   "license": "MIT",
   "description": "Local message and config persistence for Vox",
@@ -9,9 +9,15 @@
   "exports": {
     ".": "./src/index.js",
     "./db": "./src/db.js",
-    "./messages": "./src/messages.js",
-    "./config": "./src/config.js",
-    "./tasks": "./src/tasks.js"
+    "./messages": "./src/repos/messages.js",
+    "./tasks": "./src/repos/tasks.js",
+    "./tools": "./src/repos/tools.js",
+    "./settings": "./src/repos/settings.js",
+    "./mcp-servers": "./src/repos/mcp-servers.js",
+    "./schedules": "./src/repos/schedules.js",
+    "./tool-secrets": "./src/repos/tool-secrets.js",
+    "./patterns": "./src/repos/patterns.js",
+    "./vectors": "./src/repos/vectors.js"
   },
   "publishConfig": {
     "access": "public",

package/src/db.js

@@ -1,9 +1,14 @@
 import Database from 'better-sqlite3'
 import { mkdirSync } from 'node:fs'
 import path from 'node:path'
+import { runMigrations } from './migrations/runner.js'
+import * as initialSchema from './migrations/001_initial_schema.js'
+import * as taskActivityTypes from './migrations/002_task_activity_types.js'
 
 const dbs = new Map()
 
+const migrations = [initialSchema, taskActivityTypes]
+
 function resolveDbPath(dbPath) {
   const normalized = String(dbPath || '').trim()
   if (!normalized) {
@@ -11,116 +16,10 @@ function resolveDbPath(dbPath) {
   }
   return path.resolve(normalized)
 }
-
 function prepareDb(db) {
   db.pragma('journal_mode = WAL')
   db.pragma('foreign_keys = ON')
-
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS conversations (
-      id TEXT PRIMARY KEY,
-      created_at TEXT NOT NULL,
-      updated_at TEXT NOT NULL
-    );
-
-    CREATE TABLE IF NOT EXISTS messages (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      conversation_id TEXT NOT NULL,
-      role TEXT NOT NULL,
-      content TEXT NOT NULL,
-      created_at TEXT NOT NULL,
-      FOREIGN KEY (conversation_id) REFERENCES conversations(id) ON DELETE CASCADE
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_messages_conversation_id_id
-      ON messages (conversation_id, id);
-
-    CREATE TABLE IF NOT EXISTS tasks (
-      task_id TEXT PRIMARY KEY,
-      instructions TEXT NOT NULL DEFAULT '',
-      context TEXT NOT NULL DEFAULT '',
-      status TEXT NOT NULL DEFAULT 'queued',
-      created_at TEXT NOT NULL,
-      updated_at TEXT NOT NULL,
-      current_plan TEXT NOT NULL DEFAULT '',
-      message TEXT NOT NULL DEFAULT '',
-      result TEXT,
-      completed_at TEXT NOT NULL DEFAULT '',
-      failed_at TEXT NOT NULL DEFAULT ''
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_tasks_created_at
-      ON tasks (created_at DESC, task_id DESC);
-
-    CREATE TABLE IF NOT EXISTS task_activity (
-      id TEXT PRIMARY KEY,
-      task_id TEXT NOT NULL,
-      type TEXT NOT NULL,
-      name TEXT,
-      raw_result TEXT,
-      timestamp TEXT NOT NULL,
-      data TEXT NOT NULL DEFAULT '{}',
-      FOREIGN KEY (task_id) REFERENCES tasks(task_id) ON DELETE CASCADE
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_task_activity_timestamp
-      ON task_activity (timestamp ASC, id ASC);
-
-    CREATE INDEX IF NOT EXISTS idx_task_activity_task_id
-      ON task_activity (task_id, timestamp ASC, id ASC);
-  `)
-
-  try {
-    db.exec(`ALTER TABLE tasks ADD COLUMN reported INTEGER NOT NULL DEFAULT 0`)
-  } catch {
-    /* */
-  }
-
-  try {
-    db.exec(`ALTER TABLE conversations ADD COLUMN context_summary TEXT`)
-  } catch {
-    /* */
-  }
-
-  try {
-    db.exec(`ALTER TABLE conversations ADD COLUMN context_checkpoint_id INTEGER`)
-  } catch {
-    /* */
-  }
-
-  try {
-    db.exec(`
-      CREATE VIRTUAL TABLE IF NOT EXISTS tasks_fts USING fts5(
-        task_id UNINDEXED,
-        instructions,
-        result,
-        tokenize = 'unicode61'
-      )
-    `)
-  } catch {
-    /* table may already exist */
-  }
-
-  try {
-    db.exec(`
-      CREATE TABLE IF NOT EXISTS knowledge_patterns (
-        id TEXT PRIMARY KEY,
-        trigger TEXT NOT NULL,
-        solution TEXT NOT NULL,
-        created_at TEXT NOT NULL
-      )
-    `)
-    db.exec(`
-      CREATE VIRTUAL TABLE IF NOT EXISTS patterns_fts USING fts5(
-        pattern_id UNINDEXED,
-        trigger,
-        solution,
-        tokenize = 'unicode61'
-      )
-    `)
-  } catch {
-    /* tables may already exist */
-  }
+  runMigrations(db, migrations)
 }
 
 export function openDb(dbPath) {

package/src/index.js

@@ -1,4 +1,10 @@
 export * from './db.js'
-export * from './messages.js'
-export * from './config.js'
-export * from './tasks.js'
+export * from './repos/messages.js'
+export * from './repos/tasks.js'
+export * from './repos/tools.js'
+export * from './repos/settings.js'
+export * from './repos/mcp-servers.js'
+export * from './repos/schedules.js'
+export * from './repos/tool-secrets.js'
+export * from './repos/patterns.js'
+export * from './repos/vectors.js'

package/src/migrations/001_initial_schema.js

@@ -0,0 +1,217 @@
+export const name = '001_initial_schema'
+
+export function up(db) {
+  db.exec(`
+    CREATE TABLE conversations (
+      id                    TEXT    PRIMARY KEY,
+      title                 TEXT,
+      user_info             TEXT    NOT NULL DEFAULT '{}',
+      context_summary       TEXT,
+      context_checkpoint_id TEXT,
+      created_at            TEXT    NOT NULL,
+      updated_at            TEXT    NOT NULL
+    );
+
+    CREATE TABLE messages (
+      sort_order        INTEGER PRIMARY KEY AUTOINCREMENT,
+      id                TEXT    NOT NULL UNIQUE,
+      conversation_id   TEXT    NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
+      role              TEXT    NOT NULL CHECK(role IN ('user','assistant','system','tool','tool_result')),
+      content           TEXT    NOT NULL,
+      tokens            INTEGER,
+      created_at        TEXT    NOT NULL,
+      updated_at        TEXT    NOT NULL
+    );
+
+    CREATE INDEX idx_messages_conversation
+      ON messages(conversation_id, sort_order);
+
+    CREATE TABLE tasks (
+      id                TEXT    PRIMARY KEY,
+      instructions      TEXT    NOT NULL DEFAULT '',
+      context           TEXT    NOT NULL DEFAULT '',
+      status            TEXT    NOT NULL DEFAULT 'queued'
+                        CHECK(status IN ('queued','running','completed','incomplete','failed','aborted')),
+      current_plan      TEXT    NOT NULL DEFAULT '',
+      result            TEXT,
+      error             TEXT,
+      abort_reason      TEXT,
+      provider          TEXT,
+      model             TEXT,
+      context_injected  INTEGER NOT NULL DEFAULT 0,
+      created_at        TEXT    NOT NULL,
+      updated_at        TEXT    NOT NULL,
+      completed_at      TEXT
+    );
+
+    CREATE INDEX idx_tasks_created
+      ON tasks(created_at DESC, id DESC);
+
+    CREATE INDEX idx_tasks_status
+      ON tasks(status, created_at DESC);
+
+    CREATE TABLE task_activity (
+      id          TEXT    PRIMARY KEY,
+      task_id     TEXT    NOT NULL REFERENCES tasks(id) ON DELETE CASCADE,
+      type        TEXT    NOT NULL CHECK(type IN ('tool','spawn','success','error','info','step')),
+      name        TEXT,
+      result      TEXT,
+      data        TEXT    NOT NULL DEFAULT '{}',
+      created_at  TEXT    NOT NULL
+    );
+
+    CREATE INDEX idx_task_activity_task
+      ON task_activity(task_id, created_at ASC, id ASC);
+
+    CREATE VIRTUAL TABLE tasks_fts USING fts5(
+      task_id UNINDEXED,
+      instructions,
+      result,
+      tokenize = 'unicode61'
+    );
+
+    CREATE TRIGGER tasks_fts_insert AFTER INSERT ON tasks BEGIN
+      INSERT INTO tasks_fts(task_id, instructions, result)
+      VALUES (NEW.id, NEW.instructions, COALESCE(NEW.result, ''));
+    END;
+
+    CREATE TRIGGER tasks_fts_update AFTER UPDATE OF instructions, result ON tasks BEGIN
+      DELETE FROM tasks_fts WHERE task_id = OLD.id;
+      INSERT INTO tasks_fts(task_id, instructions, result)
+      VALUES (NEW.id, NEW.instructions, COALESCE(NEW.result, ''));
+    END;
+
+    CREATE TRIGGER tasks_fts_delete AFTER DELETE ON tasks BEGIN
+      DELETE FROM tasks_fts WHERE task_id = OLD.id;
+    END;
+
+    CREATE VIRTUAL TABLE messages_fts USING fts5(
+      message_id UNINDEXED,
+      role UNINDEXED,
+      content,
+      tokenize = 'unicode61'
+    );
+
+    CREATE TRIGGER messages_fts_insert AFTER INSERT ON messages BEGIN
+      INSERT INTO messages_fts(message_id, role, content)
+      VALUES (NEW.id, NEW.role, NEW.content);
+    END;
+
+    CREATE TRIGGER messages_fts_update AFTER UPDATE OF content ON messages BEGIN
+      DELETE FROM messages_fts WHERE message_id = OLD.id;
+      INSERT INTO messages_fts(message_id, role, content)
+      VALUES (NEW.id, NEW.role, NEW.content);
+    END;
+
+    CREATE TRIGGER messages_fts_delete AFTER DELETE ON messages BEGIN
+      DELETE FROM messages_fts WHERE message_id = OLD.id;
+    END;
+
+    CREATE TABLE tools (
+      id              TEXT    PRIMARY KEY,
+      name            TEXT    NOT NULL UNIQUE CHECK(length(name) BETWEEN 1 AND 64),
+      description     TEXT    NOT NULL DEFAULT '',
+      parameters      TEXT    NOT NULL DEFAULT '{"type":"object","properties":{}}',
+      source_type     TEXT    NOT NULL CHECK(source_type IN ('js_function','http_webhook','desktop')),
+      source_code     TEXT,
+      webhook_url     TEXT,
+      webhook_headers TEXT    NOT NULL DEFAULT '{}',
+      is_enabled      INTEGER NOT NULL DEFAULT 1,
+      tags            TEXT    NOT NULL DEFAULT '[]',
+      version         INTEGER NOT NULL DEFAULT 1,
+      created_at      TEXT    NOT NULL,
+      updated_at      TEXT    NOT NULL
+    );
+
+    CREATE INDEX idx_tools_enabled
+      ON tools(is_enabled) WHERE is_enabled = 1;
+
+    CREATE TABLE tool_secrets (
+      id              TEXT    PRIMARY KEY,
+      tool_id         TEXT    NOT NULL REFERENCES tools(id) ON DELETE CASCADE,
+      key             TEXT    NOT NULL,
+      encrypted_value TEXT    NOT NULL,
+      created_at      TEXT    NOT NULL,
+      updated_at      TEXT    NOT NULL,
+      UNIQUE(tool_id, key)
+    );
+
+    CREATE INDEX idx_tool_secrets_tool
+      ON tool_secrets(tool_id);
+
+    CREATE TABLE mcp_servers (
+      id              TEXT    PRIMARY KEY,
+      name            TEXT    NOT NULL,
+      transport       TEXT    NOT NULL CHECK(transport IN ('stdio','sse','http')),
+      command         TEXT,
+      args            TEXT    NOT NULL DEFAULT '[]',
+      url             TEXT,
+      env             TEXT    NOT NULL DEFAULT '{}',
+      is_enabled      INTEGER NOT NULL DEFAULT 1,
+      created_at      TEXT    NOT NULL,
+      updated_at      TEXT    NOT NULL
+    );
+
+    CREATE TABLE settings (
+      key         TEXT    PRIMARY KEY,
+      value       TEXT    NOT NULL,
+      updated_at  TEXT    NOT NULL
+    );
+
+    CREATE TABLE schedules (
+      id          TEXT    PRIMARY KEY,
+      cron_expr   TEXT    NOT NULL,
+      timezone    TEXT,
+      prompt      TEXT    NOT NULL,
+      channel     TEXT,
+      is_enabled  INTEGER NOT NULL DEFAULT 1,
+      once        INTEGER NOT NULL DEFAULT 0,
+      created_at  TEXT    NOT NULL,
+      updated_at  TEXT    NOT NULL
+    );
+
+    CREATE TABLE vectors (
+      id              TEXT    NOT NULL,
+      collection      TEXT    NOT NULL,
+      embedding       BLOB    NOT NULL,
+      metadata        TEXT    NOT NULL DEFAULT '{}',
+      created_at      TEXT    NOT NULL,
+      updated_at      TEXT    NOT NULL,
+      PRIMARY KEY (id, collection)
+    );
+
+    CREATE INDEX idx_vectors_collection
+      ON vectors(collection);
+
+    CREATE TABLE knowledge_patterns (
+      id          TEXT    PRIMARY KEY,
+      task_id     TEXT    REFERENCES tasks(id) ON DELETE SET NULL,
+      trigger     TEXT    NOT NULL,
+      solution    TEXT    NOT NULL,
+      created_at  TEXT    NOT NULL,
+      updated_at  TEXT    NOT NULL
+    );
+
+    CREATE VIRTUAL TABLE patterns_fts USING fts5(
+      pattern_id UNINDEXED,
+      trigger,
+      solution,
+      tokenize = 'unicode61'
+    );
+
+    CREATE TRIGGER patterns_fts_insert AFTER INSERT ON knowledge_patterns BEGIN
+      INSERT INTO patterns_fts(pattern_id, trigger, solution)
+      VALUES (NEW.id, NEW.trigger, NEW.solution);
+    END;
+
+    CREATE TRIGGER patterns_fts_update AFTER UPDATE OF trigger, solution ON knowledge_patterns BEGIN
+      DELETE FROM patterns_fts WHERE pattern_id = OLD.id;
+      INSERT INTO patterns_fts(pattern_id, trigger, solution)
+      VALUES (NEW.id, NEW.trigger, NEW.solution);
+    END;
+
+    CREATE TRIGGER patterns_fts_delete AFTER DELETE ON knowledge_patterns BEGIN
+      DELETE FROM patterns_fts WHERE pattern_id = OLD.id;
+    END;
+  `)
+}

package/src/migrations/002_task_activity_types.js

@@ -0,0 +1,45 @@
+export const name = '002_task_activity_types'
+
+export function up(db) {
+  db.exec(`
+    CREATE TABLE task_activity_new (
+      id          TEXT PRIMARY KEY,
+      task_id     TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE,
+      type        TEXT NOT NULL CHECK(type IN (
+        'tool_call','tool_result','text','thought',
+        'journal','spawn','error'
+      )),
+      name        TEXT,
+      args        TEXT,
+      result      TEXT,
+      plan        TEXT,
+      data        TEXT NOT NULL DEFAULT '{}',
+      created_at  TEXT NOT NULL
+    );
+
+    INSERT INTO task_activity_new (id, task_id, type, name, args, result, plan, data, created_at)
+    SELECT
+      id,
+      task_id,
+      CASE type
+        WHEN 'tool'    THEN 'tool_call'
+        WHEN 'step'    THEN 'journal'
+        WHEN 'info'    THEN 'text'
+        WHEN 'success' THEN 'text'
+        ELSE type
+      END,
+      name,
+      NULL,
+      result,
+      NULL,
+      data,
+      created_at
+    FROM task_activity;
+
+    DROP TABLE task_activity;
+    ALTER TABLE task_activity_new RENAME TO task_activity;
+
+    CREATE INDEX idx_task_activity_task
+      ON task_activity(task_id, created_at ASC, id ASC);
+  `)
+}

package/src/migrations/runner.js

@@ -0,0 +1,31 @@
+export function runMigrations(db, migrations) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS _migrations (
+      id          INTEGER PRIMARY KEY AUTOINCREMENT,
+      name        TEXT    NOT NULL UNIQUE,
+      applied_at  TEXT    NOT NULL
+    )
+  `)
+
+  const applied = new Set(
+    db
+      .prepare('SELECT name FROM _migrations')
+      .all()
+      .map((r) => r.name)
+  )
+
+  const pending = migrations
+    .slice()
+    .sort((a, b) => a.name.localeCompare(b.name))
+    .filter((m) => !applied.has(m.name))
+
+  for (const migration of pending) {
+    db.transaction(() => {
+      migration.up(db)
+      db.prepare('INSERT INTO _migrations (name, applied_at) VALUES (?, ?)').run(
+        migration.name,
+        new Date().toISOString()
+      )
+    })()
+  }
+}

package/src/repos/mcp-servers.js

@@ -0,0 +1,81 @@
+import { randomUUID } from 'crypto'
+
+function mapServer(row) {
+  if (!row) return null
+  return {
+    id: row.id,
+    name: row.name,
+    transport: row.transport,
+    command: row.command || null,
+    args: JSON.parse(row.args),
+    url: row.url || null,
+    env: JSON.parse(row.env),
+    isEnabled: !!row.is_enabled,
+    createdAt: row.created_at,
+    updatedAt: row.updated_at
+  }
+}
+
+export function createMcpServer(db, server) {
+  const id = server.id || randomUUID()
+  const now = new Date().toISOString()
+
+  db.prepare(
+    `INSERT INTO mcp_servers (id, name, transport, command, args, url, env, is_enabled, created_at, updated_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+  ).run(
+    id,
+    String(server.name),
+    String(server.transport || 'stdio'),
+    server.command || null,
+    JSON.stringify(server.args || []),
+    server.url || null,
+    JSON.stringify(server.env || {}),
+    server.isEnabled === false ? 0 : 1,
+    now,
+    now
+  )
+
+  return getMcpServer(db, id)
+}
+
+export function updateMcpServer(db, id, updates) {
+  const existing = getMcpServer(db, id)
+  if (!existing) return null
+
+  const now = new Date().toISOString()
+  const merged = { ...existing, ...updates }
+
+  db.prepare(
+    `UPDATE mcp_servers SET name = ?, transport = ?, command = ?, args = ?, url = ?,
+      env = ?, is_enabled = ?, updated_at = ?
+     WHERE id = ?`
+  ).run(
+    String(merged.name),
+    String(merged.transport || 'stdio'),
+    merged.command || null,
+    JSON.stringify(merged.args || []),
+    merged.url || null,
+    JSON.stringify(merged.env || {}),
+    merged.isEnabled === false ? 0 : 1,
+    now,
+    id
+  )
+
+  return getMcpServer(db, id)
+}
+
+export function deleteMcpServer(db, id) {
+  return db.prepare(`DELETE FROM mcp_servers WHERE id = ?`).run(id)
+}
+
+export function getMcpServer(db, id) {
+  return mapServer(db.prepare(`SELECT * FROM mcp_servers WHERE id = ?`).get(id))
+}
+
+export function listMcpServers(db, enabledOnly = false) {
+  if (enabledOnly) {
+    return db.prepare(`SELECT * FROM mcp_servers WHERE is_enabled = 1`).all().map(mapServer)
+  }
+  return db.prepare(`SELECT * FROM mcp_servers ORDER BY name ASC`).all().map(mapServer)
+}