@arabold/docs-mcp-server 1.17.0 → 1.19.0

package/db/migrations/004-complete-normalization.sql

@@ -0,0 +1,67 @@
+-- Migration: Complete normalization by removing obsolete library and version columns
+-- This migration finalizes the schema normalization process
+-- Note: Must recreate table because obsolete columns are part of UNIQUE constraint
+
+-- 1. Create new documents table with only foreign key references
+CREATE TABLE documents_new (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  library_id INTEGER NOT NULL REFERENCES libraries(id),
+  version_id INTEGER NOT NULL REFERENCES versions(id),
+  url TEXT NOT NULL,
+  content TEXT,
+  metadata JSON,
+  sort_order INTEGER NOT NULL,
+  indexed_at DATETIME,
+  UNIQUE(url, library_id, version_id, sort_order)
+);
+
+-- 2. Copy data from old table (excluding obsolete library and version columns)
+INSERT INTO documents_new (id, library_id, version_id, url, content, metadata, sort_order, indexed_at)
+SELECT id, library_id, version_id, url, content, metadata, sort_order, indexed_at
+FROM documents;
+
+-- 3. Drop the old documents table
+DROP TABLE documents;
+
+-- 4. Rename the new table to documents
+ALTER TABLE documents_new RENAME TO documents;
+
+-- 5. Recreate indexes that were lost when dropping the table
+CREATE INDEX IF NOT EXISTS idx_documents_library_id ON documents(library_id);
+CREATE INDEX IF NOT EXISTS idx_documents_version_id ON documents(version_id);
+CREATE INDEX IF NOT EXISTS idx_documents_lib_ver_id ON documents(library_id, version_id);
+
+-- 6. Recreate FTS5 virtual table (gets dropped when main table is dropped)
+-- Using external content approach - FTS index is maintained entirely through triggers
+CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(
+  content,
+  title,
+  url,
+  path,
+  tokenize='porter unicode61'
+);
+
+-- 7. Recreate FTS triggers to maintain the index
+-- Note: Triggers work directly with documents table, no JOIN needed for FTS content
+CREATE TRIGGER IF NOT EXISTS documents_fts_after_delete AFTER DELETE ON documents BEGIN
+  INSERT INTO documents_fts(documents_fts, rowid, content, title, url, path)
+  VALUES('delete', old.id, old.content, json_extract(old.metadata, '$.title'), old.url, json_extract(old.metadata, '$.path'));
+END;
+
+CREATE TRIGGER IF NOT EXISTS documents_fts_after_update AFTER UPDATE ON documents BEGIN
+  INSERT INTO documents_fts(documents_fts, rowid, content, title, url, path)
+  VALUES('delete', old.id, old.content, json_extract(old.metadata, '$.title'), old.url, json_extract(old.metadata, '$.path'));
+  INSERT INTO documents_fts(rowid, content, title, url, path)
+  VALUES(new.id, new.content, json_extract(new.metadata, '$.title'), new.url, json_extract(new.metadata, '$.path'));
+END;
+
+CREATE TRIGGER IF NOT EXISTS documents_fts_after_insert AFTER INSERT ON documents BEGIN
+  INSERT INTO documents_fts(rowid, content, title, url, path)
+  VALUES(new.id, new.content, json_extract(new.metadata, '$.title'), new.url, json_extract(new.metadata, '$.path'));
+END;
+
+-- 8. Rebuild FTS index from existing documents data
+-- Manually populate the FTS index since we're using external content approach
+INSERT INTO documents_fts(rowid, content, title, url, path)
+SELECT id, content, json_extract(metadata, '$.title'), url, json_extract(metadata, '$.path')
+FROM documents;

package/db/migrations/005-add-status-tracking.sql

@@ -0,0 +1,42 @@
+-- Migration: Add status tracking and progress monitoring to versions table
+-- This migration adds job status tracking directly to the versions table
+-- enabling persistent job state and progress monitoring across server restarts
+
+-- Add job status and progress tracking columns
+ALTER TABLE versions ADD COLUMN status TEXT DEFAULT 'not_indexed';
+ALTER TABLE versions ADD COLUMN progress_pages INTEGER DEFAULT 0;
+ALTER TABLE versions ADD COLUMN progress_max_pages INTEGER DEFAULT 0;
+ALTER TABLE versions ADD COLUMN error_message TEXT;
+ALTER TABLE versions ADD COLUMN started_at DATETIME;
+ALTER TABLE versions ADD COLUMN updated_at DATETIME;
+
+-- Create indexes for efficient status queries
+CREATE INDEX IF NOT EXISTS idx_versions_status ON versions(status);
+CREATE INDEX IF NOT EXISTS idx_versions_started_at ON versions(started_at);
+CREATE INDEX IF NOT EXISTS idx_versions_library_status ON versions(library_id, status);
+
+-- Create trigger to automatically update updated_at timestamp
+CREATE TRIGGER IF NOT EXISTS versions_updated_at
+AFTER UPDATE ON versions BEGIN
+  UPDATE versions SET updated_at = CURRENT_TIMESTAMP WHERE id = NEW.id;
+END;
+
+-- Set existing versions to 'completed' status based on whether they have documents
+-- This ensures backward compatibility with existing data
+UPDATE versions 
+SET status = 'completed', updated_at = CURRENT_TIMESTAMP
+WHERE id IN (
+  SELECT DISTINCT v.id 
+  FROM versions v 
+  JOIN documents d ON v.id = d.version_id
+);
+
+-- Set updated_at for any remaining records without it
+UPDATE versions 
+SET updated_at = CURRENT_TIMESTAMP
+WHERE updated_at IS NULL;
+
+-- Note: Versions without documents remain as 'not_indexed' which is correct
+-- as they were created but never successfully indexed
+-- The started_at field tracks when indexing jobs begin
+-- The status field provides comprehensive state tracking for version indexing

package/db/migrations/006-add-scraper-options.sql

@@ -0,0 +1,16 @@
+-- Migration: Add scraper options tracking to versions table
+-- This migration adds scraper options storage to enable reproducible indexing
+-- with the exact same parameters used in previous runs
+
+-- Add scraper options tracking columns
+ALTER TABLE versions ADD COLUMN source_url TEXT;
+ALTER TABLE versions ADD COLUMN scraper_options JSON;
+
+-- Create indexes for efficient queries
+CREATE INDEX IF NOT EXISTS idx_versions_source_url ON versions(source_url);
+CREATE INDEX IF NOT EXISTS idx_versions_scraper_options_scope 
+ON versions(json_extract(scraper_options, '$.scope'));
+
+-- Note: No data migration needed - new columns default to NULL
+-- Existing versions without stored options will gracefully fallback to manual configuration
+-- Future indexing operations will store complete scraper options for reproducibility

package/db/migrations/007-dedupe-unversioned-versions.sql

@@ -0,0 +1,62 @@
+-- Migration 007: Deduplicate unversioned versions
+-- Goal: collapse multiple NULL-name version rows per library to a single canonical row
+-- Steps:
+-- 1. For each library, choose canonical NULL-name row:
+--    a) Prefer a row referenced by any documents (highest document count)
+--    b) Fallback to lowest id
+-- 2. Repoint any documents referencing non-canonical NULL rows to canonical
+-- 3. Delete surplus NULL-name rows with zero documents
+-- 4. Convert remaining NULL names to empty string '' for future uniqueness enforcement
+-- Safe to run multiple times (idempotent)
+
+-- 1 & 2: Repoint documents
+-- Use TEMP tables instead of CTEs because we need the canonical mapping
+-- across multiple subsequent statements. All TEMP objects are connection-scoped
+-- and vanish automatically; safe for repeated runs (we DROP IF EXISTS first).
+
+DROP TABLE IF EXISTS temp_null_versions;
+CREATE TEMP TABLE temp_null_versions AS
+SELECT v.id, v.library_id,
+       (SELECT COUNT(*) FROM documents d WHERE d.version_id = v.id) AS doc_count
+FROM versions v
+WHERE v.name IS NULL;
+
+-- Build canonical mapping per library (one row per library_id)
+DROP TABLE IF EXISTS temp_canonical_versions;
+CREATE TEMP TABLE temp_canonical_versions AS
+SELECT nv.library_id,
+       COALESCE(
+         (
+           SELECT id FROM temp_null_versions nv2
+           WHERE nv2.library_id = nv.library_id AND nv2.doc_count > 0
+           ORDER BY nv2.doc_count DESC, nv2.id ASC LIMIT 1
+         ),
+         (
+           SELECT id FROM temp_null_versions nv3
+           WHERE nv3.library_id = nv.library_id
+           ORDER BY nv3.id ASC LIMIT 1
+         )
+       ) AS keep_id
+FROM temp_null_versions nv
+GROUP BY nv.library_id;
+
+-- Repoint documents from non-canonical NULL-name versions
+UPDATE documents
+SET version_id = (
+  SELECT keep_id FROM temp_canonical_versions c
+  WHERE c.library_id = documents.library_id
+)
+WHERE version_id IN (SELECT id FROM versions WHERE name IS NULL)
+  AND version_id NOT IN (SELECT keep_id FROM temp_canonical_versions);
+
+-- 3: Delete surplus NULL-name rows now unreferenced
+DELETE FROM versions
+WHERE name IS NULL
+  AND id NOT IN (SELECT keep_id FROM temp_canonical_versions)
+  AND (SELECT COUNT(*) FROM documents d WHERE d.version_id = versions.id) = 0;
+
+-- 4: Normalize remaining NULL names to ''
+UPDATE versions SET name = '' WHERE name IS NULL;
+
+-- (Optional) Unique index already exists if schema defined; if not, we could add:
+-- CREATE UNIQUE INDEX IF NOT EXISTS idx_versions_library_name ON versions(library_id, name);

package/db/migrations/008-case-insensitive-names.sql

@@ -0,0 +1,10 @@
+-- We only need to normalize existing strings to lower-case and add expression unique indexes
+-- for defense-in-depth. Idempotent: LOWER(name) is stable on re-run.
+
+UPDATE libraries SET name = LOWER(name);
+UPDATE versions SET name = LOWER(name) WHERE name IS NOT NULL AND name <> '';
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_libraries_lower_name ON libraries(LOWER(name));
+CREATE UNIQUE INDEX IF NOT EXISTS idx_versions_library_lower_name ON versions(library_id, LOWER(name));
+
+-- Existing UNIQUE(library_id, name) plus these expression indexes enforce case-insensitive uniqueness.

package/dist/DocumentManagementClient-CAFdDwTu.js

@@ -0,0 +1,57 @@
+import { createTRPCProxyClient, httpBatchLink } from "@trpc/client";
+import { l as logger } from "./index.js";
+class DocumentManagementClient {
+  baseUrl;
+  client;
+  constructor(serverUrl) {
+    this.baseUrl = serverUrl.replace(/\/$/, "");
+    this.client = createTRPCProxyClient({
+      links: [httpBatchLink({ url: this.baseUrl })]
+    });
+    logger.debug(`DocumentManagementClient (tRPC) created for: ${this.baseUrl}`);
+  }
+  async initialize() {
+    await this.client.ping.query();
+  }
+  async shutdown() {
+  }
+  async listLibraries() {
+    return this.client.listLibraries.query();
+  }
+  async validateLibraryExists(library) {
+    await this.client.validateLibraryExists.mutate({ library });
+  }
+  async findBestVersion(library, targetVersion) {
+    return this.client.findBestVersion.query({ library, targetVersion });
+  }
+  async searchStore(library, version, query, limit) {
+    return this.client.search.query({ library, version: version ?? null, query, limit });
+  }
+  async removeAllDocuments(library, version) {
+    await this.client.removeAllDocuments.mutate({ library, version: version ?? null });
+  }
+  async getVersionsByStatus(statuses) {
+    return this.client.getVersionsByStatus.query({
+      statuses
+    });
+  }
+  async findVersionsBySourceUrl(url) {
+    return this.client.findVersionsBySourceUrl.query({ url });
+  }
+  async getScraperOptions(versionId) {
+    return this.client.getScraperOptions.query({ versionId });
+  }
+  async updateVersionStatus(versionId, status, errorMessage) {
+    await this.client.updateVersionStatus.mutate({ versionId, status, errorMessage });
+  }
+  async updateVersionProgress(versionId, pages, maxPages) {
+    await this.client.updateVersionProgress.mutate({ versionId, pages, maxPages });
+  }
+  async storeScraperOptions(versionId, options) {
+    await this.client.storeScraperOptions.mutate({ versionId, options });
+  }
+}
+export {
+  DocumentManagementClient
+};
+//# sourceMappingURL=DocumentManagementClient-CAFdDwTu.js.map

package/dist/DocumentManagementClient-CAFdDwTu.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"DocumentManagementClient-CAFdDwTu.js","sources":["../src/store/DocumentManagementClient.ts"],"sourcesContent":["/**\n * tRPC client for the document management API.\n * Implements IDocumentManagement and delegates to /api data router.\n */\nimport { createTRPCProxyClient, httpBatchLink } from \"@trpc/client\";\nimport type { ScraperOptions } from \"../scraper/types\";\nimport { logger } from \"../utils/logger\";\nimport type { IDocumentManagement } from \"./trpc/interfaces\";\nimport type { DataRouter } from \"./trpc/router\";\nimport type {\n  DbVersionWithLibrary,\n  FindVersionResult,\n  LibrarySummary,\n  StoredScraperOptions,\n  StoreSearchResult,\n  VersionStatus,\n} from \"./types\";\n\nexport class DocumentManagementClient implements IDocumentManagement {\n  private readonly baseUrl: string;\n  private readonly client: ReturnType<typeof createTRPCProxyClient<DataRouter>>;\n\n  constructor(serverUrl: string) {\n    this.baseUrl = serverUrl.replace(/\\/$/, \"\");\n    this.client = createTRPCProxyClient<DataRouter>({\n      links: [httpBatchLink({ url: this.baseUrl })],\n    });\n    logger.debug(`DocumentManagementClient (tRPC) created for: ${this.baseUrl}`);\n  }\n\n  async initialize(): Promise<void> {\n    // Connectivity check\n    await (\n      this.client as unknown as { ping: { query: () => Promise<unknown> } }\n    ).ping.query();\n  }\n\n  async shutdown(): Promise<void> {\n    // no-op for HTTP client\n  }\n\n  async listLibraries(): Promise<LibrarySummary[]> {\n    return this.client.listLibraries.query();\n  }\n\n  async validateLibraryExists(library: string): Promise<void> {\n    await this.client.validateLibraryExists.mutate({ library });\n  }\n\n  async findBestVersion(\n    library: string,\n    targetVersion?: string,\n  ): Promise<FindVersionResult> {\n    return this.client.findBestVersion.query({ library, targetVersion });\n  }\n\n  async searchStore(\n    library: string,\n    version: string | null | undefined,\n    query: string,\n    limit?: number,\n  ): Promise<StoreSearchResult[]> {\n    return this.client.search.query({ library, version: version ?? null, query, limit });\n  }\n\n  async removeAllDocuments(library: string, version?: string | null): Promise<void> {\n    await this.client.removeAllDocuments.mutate({ library, version: version ?? null });\n  }\n\n  async getVersionsByStatus(statuses: VersionStatus[]): Promise<DbVersionWithLibrary[]> {\n    return this.client.getVersionsByStatus.query({\n      statuses: statuses as unknown as string[],\n    });\n  }\n\n  async findVersionsBySourceUrl(url: string): Promise<DbVersionWithLibrary[]> {\n    return this.client.findVersionsBySourceUrl.query({ url });\n  }\n\n  async getScraperOptions(versionId: number): Promise<StoredScraperOptions | null> {\n    return this.client.getScraperOptions.query({ versionId });\n  }\n\n  async updateVersionStatus(\n    versionId: number,\n    status: VersionStatus,\n    errorMessage?: string,\n  ): Promise<void> {\n    await this.client.updateVersionStatus.mutate({ versionId, status, errorMessage });\n  }\n\n  async updateVersionProgress(\n    versionId: number,\n    pages: number,\n    maxPages: number,\n  ): Promise<void> {\n    await this.client.updateVersionProgress.mutate({ versionId, pages, maxPages });\n  }\n\n  async storeScraperOptions(versionId: number, options: ScraperOptions): Promise<void> {\n    await this.client.storeScraperOptions.mutate({ versionId, options });\n  }\n}\n"],"names":[],"mappings":";;AAkBO,MAAM,yBAAwD;AAAA,EAClD;AAAA,EACA;AAAA,EAEjB,YAAY,WAAmB;AAC7B,SAAK,UAAU,UAAU,QAAQ,OAAO,EAAE;AAC1C,SAAK,SAAS,sBAAkC;AAAA,MAC9C,OAAO,CAAC,cAAc,EAAE,KAAK,KAAK,QAAA,CAAS,CAAC;AAAA,IAAA,CAC7C;AACD,WAAO,MAAM,gDAAgD,KAAK,OAAO,EAAE;AAAA,EAC7E;AAAA,EAEA,MAAM,aAA4B;AAEhC,UACE,KAAK,OACL,KAAK,MAAA;AAAA,EACT;AAAA,EAEA,MAAM,WAA0B;AAAA,EAEhC;AAAA,EAEA,MAAM,gBAA2C;AAC/C,WAAO,KAAK,OAAO,cAAc,MAAA;AAAA,EACnC;AAAA,EAEA,MAAM,sBAAsB,SAAgC;AAC1D,UAAM,KAAK,OAAO,sBAAsB,OAAO,EAAE,SAAS;AAAA,EAC5D;AAAA,EAEA,MAAM,gBACJ,SACA,eAC4B;AAC5B,WAAO,KAAK,OAAO,gBAAgB,MAAM,EAAE,SAAS,eAAe;AAAA,EACrE;AAAA,EAEA,MAAM,YACJ,SACA,SACA,OACA,OAC8B;AAC9B,WAAO,KAAK,OAAO,OAAO,MAAM,EAAE,SAAS,SAAS,WAAW,MAAM,OAAO,MAAA,CAAO;AAAA,EACrF;AAAA,EAEA,MAAM,mBAAmB,SAAiB,SAAwC;AAChF,UAAM,KAAK,OAAO,mBAAmB,OAAO,EAAE,SAAS,SAAS,WAAW,MAAM;AAAA,EACnF;AAAA,EAEA,MAAM,oBAAoB,UAA4D;AACpF,WAAO,KAAK,OAAO,oBAAoB,MAAM;AAAA,MAC3C;AAAA,IAAA,CACD;AAAA,EACH;AAAA,EAEA,MAAM,wBAAwB,KAA8C;AAC1E,WAAO,KAAK,OAAO,wBAAwB,MAAM,EAAE,KAAK;AAAA,EAC1D;AAAA,EAEA,MAAM,kBAAkB,WAAyD;AAC/E,WAAO,KAAK,OAAO,kBAAkB,MAAM,EAAE,WAAW;AAAA,EAC1D;AAAA,EAEA,MAAM,oBACJ,WACA,QACA,cACe;AACf,UAAM,KAAK,OAAO,oBAAoB,OAAO,EAAE,WAAW,QAAQ,cAAc;AAAA,EAClF;AAAA,EAEA,MAAM,sBACJ,WACA,OACA,UACe;AACf,UAAM,KAAK,OAAO,sBAAsB,OAAO,EAAE,WAAW,OAAO,UAAU;AAAA,EAC/E;AAAA,EAEA,MAAM,oBAAoB,WAAmB,SAAwC;AACnF,UAAM,KAAK,OAAO,oBAAoB,OAAO,EAAE,WAAW,SAAS;AAAA,EACrE;AACF;"}