@talkpilot/core-db 1.3.47 → 1.3.49

package/src/websitalk/knowledgeDocuments/knowledgeDocuments.types.ts

@@ -1,62 +1,62 @@
-import { WithId, ObjectId as MongoObjectId } from "mongodb";
-
-import type {
-  KNOWLEDGE_CONTENT_TYPES,
-  KNOWLEDGE_DOCUMENT_STATUSES,
-  KNOWLEDGE_FILE_TYPES,
-} from "./knowledgeDocuments.constants";
-
-export type KnowledgeDocumentStatus =
-  (typeof KNOWLEDGE_DOCUMENT_STATUSES)[keyof typeof KNOWLEDGE_DOCUMENT_STATUSES];
-
-export type KnowledgeFileType =
-  (typeof KNOWLEDGE_FILE_TYPES)[keyof typeof KNOWLEDGE_FILE_TYPES];
-
-export type KnowledgeContentType =
-  (typeof KNOWLEDGE_CONTENT_TYPES)[keyof typeof KNOWLEDGE_CONTENT_TYPES];
-
-export type KnowledgeDocument = {
-  clientId: string;
-  filename: string;
-  contentType: KnowledgeContentType;
-  fileType: KnowledgeFileType;
-  storagePath: string;
-  markdownStoragePath?: string;
-  byteSize: number;
-  pageCount?: number;
-  pagesExtracted?: number;
-  charCount?: number;
-  contentHash?: string;
-  status: KnowledgeDocumentStatus;
-  errorMessage?: string;
-  progressPercent: number;
-  createdAt: Date;
-  updatedAt: Date;
-};
-
-export type KnowledgeDocumentDoc = WithId<KnowledgeDocument>;
-
-export type CreateKnowledgeDocumentInput = Omit<
-  KnowledgeDocument,
-  "createdAt" | "updatedAt" | "progressPercent" | "status"
-> & {
-  status?: KnowledgeDocumentStatus;
-  progressPercent?: number;
-  _id?: MongoObjectId;
-};
-
-export type UpdateKnowledgeDocumentInput = Partial<
-  Pick<
-    KnowledgeDocument,
-    | "storagePath"
-    | "markdownStoragePath"
-    | "byteSize"
-    | "pageCount"
-    | "pagesExtracted"
-    | "charCount"
-    | "contentHash"
-    | "status"
-    | "errorMessage"
-    | "progressPercent"
-  >
->;
+import { WithId, ObjectId as MongoObjectId } from "mongodb";
+
+import type {
+  KNOWLEDGE_CONTENT_TYPES,
+  KNOWLEDGE_DOCUMENT_STATUSES,
+  KNOWLEDGE_FILE_TYPES,
+} from "./knowledgeDocuments.constants";
+
+export type KnowledgeDocumentStatus =
+  (typeof KNOWLEDGE_DOCUMENT_STATUSES)[keyof typeof KNOWLEDGE_DOCUMENT_STATUSES];
+
+export type KnowledgeFileType =
+  (typeof KNOWLEDGE_FILE_TYPES)[keyof typeof KNOWLEDGE_FILE_TYPES];
+
+export type KnowledgeContentType =
+  (typeof KNOWLEDGE_CONTENT_TYPES)[keyof typeof KNOWLEDGE_CONTENT_TYPES];
+
+export type KnowledgeDocument = {
+  clientId: string;
+  filename: string;
+  contentType: KnowledgeContentType;
+  fileType: KnowledgeFileType;
+  storagePath: string;
+  markdownStoragePath?: string;
+  byteSize: number;
+  pageCount?: number;
+  pagesExtracted?: number;
+  charCount?: number;
+  contentHash?: string;
+  status: KnowledgeDocumentStatus;
+  errorMessage?: string;
+  progressPercent: number;
+  createdAt: Date;
+  updatedAt: Date;
+};
+
+export type KnowledgeDocumentDoc = WithId<KnowledgeDocument>;
+
+export type CreateKnowledgeDocumentInput = Omit<
+  KnowledgeDocument,
+  "createdAt" | "updatedAt" | "progressPercent" | "status"
+> & {
+  status?: KnowledgeDocumentStatus;
+  progressPercent?: number;
+  _id?: MongoObjectId;
+};
+
+export type UpdateKnowledgeDocumentInput = Partial<
+  Pick<
+    KnowledgeDocument,
+    | "storagePath"
+    | "markdownStoragePath"
+    | "byteSize"
+    | "pageCount"
+    | "pagesExtracted"
+    | "charCount"
+    | "contentHash"
+    | "status"
+    | "errorMessage"
+    | "progressPercent"
+  >
+>;

package/src/websitalk/mongodb-client.ts

@@ -1,61 +1,61 @@
-import { MongoClient, Db } from "mongodb";
-import { setDb } from "./index";
-import { validateConfig, validateMongoUri } from "../utils/validation";
-
-class WebsitalkMongoDBClient {
-  private client: MongoClient | null = null;
-  private db: Db | null = null;
-  private readonly defaultDbName = "website-talk";
-
-  async connect(uri?: string, dbName?: string): Promise<void> {
-    if (this.client) {
-      return;
-    }
-
-    const mongodbUri = uri || process.env.MONGO_URI || process.env.MONGODB_URI;
-    validateConfig("MONGO_URI", mongodbUri);
-    validateMongoUri(mongodbUri!);
-
-    try {
-      this.client = new MongoClient(mongodbUri!);
-      await this.client.connect();
-      const targetDbName =
-        dbName || process.env.WEBSITALK_DB_NAME || this.defaultDbName;
-      this.db = this.client.db(targetDbName);
-      setDb(this.db);
-      console.info(`[core-db] Website Talk MongoDB connected: ${targetDbName}`);
-    } catch (error) {
-      console.error("[core-db] Website Talk connection failed", error);
-      throw error;
-    }
-  }
-
-  async disconnect(): Promise<void> {
-    if (this.client) {
-      try {
-        await this.client.close();
-        this.client = null;
-        this.db = null;
-        console.info("Website Talk MongoDB disconnected successfully");
-      } catch (error) {
-        console.error("[core-db] Website Talk disconnection failed", error);
-        throw error;
-      }
-    }
-  }
-
-  getDb(): Db {
-    if (!this.db) {
-      throw new Error(
-        "Website Talk database not initialized. Call connect() first.",
-      );
-    }
-    return this.db;
-  }
-
-  isConnected(): boolean {
-    return this.client !== null && this.client !== undefined;
-  }
-}
-
-export const websitalkMongodbClient = new WebsitalkMongoDBClient();
+import { MongoClient, Db } from "mongodb";
+import { setDb } from "./index";
+import { validateConfig, validateMongoUri } from "../utils/validation";
+
+class WebsitalkMongoDBClient {
+  private client: MongoClient | null = null;
+  private db: Db | null = null;
+  private readonly defaultDbName = "website-talk";
+
+  async connect(uri?: string, dbName?: string): Promise<void> {
+    if (this.client) {
+      return;
+    }
+
+    const mongodbUri = uri || process.env.MONGO_URI || process.env.MONGODB_URI;
+    validateConfig("MONGO_URI", mongodbUri);
+    validateMongoUri(mongodbUri!);
+
+    try {
+      this.client = new MongoClient(mongodbUri!);
+      await this.client.connect();
+      const targetDbName =
+        dbName || process.env.WEBSITALK_DB_NAME || this.defaultDbName;
+      this.db = this.client.db(targetDbName);
+      setDb(this.db);
+      console.info(`[core-db] Website Talk MongoDB connected: ${targetDbName}`);
+    } catch (error) {
+      console.error("[core-db] Website Talk connection failed", error);
+      throw error;
+    }
+  }
+
+  async disconnect(): Promise<void> {
+    if (this.client) {
+      try {
+        await this.client.close();
+        this.client = null;
+        this.db = null;
+        console.info("Website Talk MongoDB disconnected successfully");
+      } catch (error) {
+        console.error("[core-db] Website Talk disconnection failed", error);
+        throw error;
+      }
+    }
+  }
+
+  getDb(): Db {
+    if (!this.db) {
+      throw new Error(
+        "Website Talk database not initialized. Call connect() first.",
+      );
+    }
+    return this.db;
+  }
+
+  isConnected(): boolean {
+    return this.client !== null && this.client !== undefined;
+  }
+}
+
+export const websitalkMongodbClient = new WebsitalkMongoDBClient();

package/src/websitalk/scanResources/__tests__/scanResources.spec.ts

@@ -1,136 +1,193 @@
-import { ObjectId } from "mongodb";
-import {
-  getScanResourcesCollection,
-  getScanKpiCounts,
-  getScanPages,
-  getScanResourceById,
-  getScanResourceCountsByScanIds,
-} from "../scanResources.getters";
-import { boardColumn } from "../scanResources.utils";
-import { createScanResourceFixture } from "../../../test-utils/factories";
-
-describe("db.websitalk.scanResources", () => {
-  describe("boardColumn()", () => {
-    it("folds URL_FAILED into FAILED, not READY_TO_PROCESS", () => {
-      expect(boardColumn("URL_FAILED")).toBe("FAILED");
-    });
-
-    it("folds URL_PERMANENT_FAILURE into FAILED", () => {
-      expect(boardColumn("URL_PERMANENT_FAILURE")).toBe("FAILED");
-    });
-
-    it("folds duplicate content into COMPLETED", () => {
-      expect(boardColumn("URL_DUPLICATE_CONTENT")).toBe("COMPLETED");
-    });
-  });
-
-  describe("getScanKpiCounts()", () => {
-    it("sums to the scan's total task count and omits zero-count statuses", async () => {
-      const scanId = new ObjectId().toString();
-      await getScanResourcesCollection().insertMany([
-        createScanResourceFixture({ scanId, status: "URL_COMPLETED" }),
-        createScanResourceFixture({ scanId, status: "URL_COMPLETED" }),
-        createScanResourceFixture({ scanId, status: "URL_PERMANENT_FAILURE" }),
-      ]);
-
-      const counts = await getScanKpiCounts(scanId);
-
-      expect(counts.URL_COMPLETED).toBe(2);
-      expect(counts.URL_PERMANENT_FAILURE).toBe(1);
-      expect(counts.URL_PENDING).toBeUndefined();
-      const total = Object.values(counts).reduce((sum, n) => sum + (n ?? 0), 0);
-      expect(total).toBe(3);
-    });
-  });
-
-  describe("getScanPages()", () => {
-    it("never returns context or stageLogs, and paginates within one column", async () => {
-      const scanId = new ObjectId().toString();
-      await getScanResourcesCollection().insertMany(
-        Array.from({ length: 3 }, () =>
-          createScanResourceFixture({
-            scanId,
-            status: "URL_COMPLETED",
-            context: "heavy markdown content",
-            stageLogs: [{ from: null, to: "URL_COMPLETED", occurredAt: new Date(), details: {} }],
-          }),
-        ),
-      );
-
-      const page1 = await getScanPages({ scanId, column: "COMPLETED", limit: 2 });
-      expect(page1.items).toHaveLength(2);
-      expect(page1.nextCursor).not.toBeNull();
-      page1.items.forEach((item) => {
-        expect(item).not.toHaveProperty("context");
-        expect(item).not.toHaveProperty("stageLogs");
-      });
-
-      const page2 = await getScanPages({
-        scanId,
-        column: "COMPLETED",
-        limit: 2,
-        cursor: page1.nextCursor,
-      });
-      expect(page2.items).toHaveLength(1);
-      expect(page2.nextCursor).toBeNull();
-    });
-
-    it("folds URL_FAILED into the FAILED column query", async () => {
-      const scanId = new ObjectId().toString();
-      await getScanResourcesCollection().insertMany([
-        createScanResourceFixture({ scanId, status: "URL_READY_TO_PROCESS" }),
-        createScanResourceFixture({ scanId, status: "URL_FAILED" }),
-        createScanResourceFixture({ scanId, status: "URL_PERMANENT_FAILURE" }),
-      ]);
-
-      const ready = await getScanPages({ scanId, column: "READY_TO_PROCESS" });
-      expect(ready.items).toHaveLength(1);
-
-      const failed = await getScanPages({ scanId, column: "FAILED" });
-      expect(failed.items).toHaveLength(2);
-    });
-  });
-
-  describe("getScanResourceCountsByScanIds()", () => {
-    it("returns per-scan totals and omits scans with no resources", async () => {
-      const scanA = new ObjectId().toString();
-      const scanB = new ObjectId().toString();
-      await getScanResourcesCollection().insertMany([
-        createScanResourceFixture({ scanId: scanA, status: "URL_COMPLETED" }),
-        createScanResourceFixture({ scanId: scanA, status: "URL_COMPLETED" }),
-        createScanResourceFixture({ scanId: scanB, status: "URL_PENDING" }),
-      ]);
-
-      const counts = await getScanResourceCountsByScanIds([scanA, scanB, "missing"]);
-      expect(counts[scanA]).toBe(2);
-      expect(counts[scanB]).toBe(1);
-      expect(counts.missing).toBeUndefined();
-    });
-  });
-
-  describe("getScanResourceById()", () => {
-    it("returns the full document including stageLogs in insertion order", async () => {
-      const scanId = new ObjectId().toString();
-      const stageLogs = [
-        { from: null, to: "URL_PENDING" as const, occurredAt: new Date("2024-01-01"), details: {} },
-        { from: "URL_PENDING" as const, to: "URL_SCANNING" as const, occurredAt: new Date("2024-01-02"), details: {} },
-      ];
-      const fixture = createScanResourceFixture({ scanId, stageLogs, context: "full content" });
-      const { insertedId } = await getScanResourcesCollection().insertOne(fixture);
-
-      const result = await getScanResourceById(scanId, insertedId.toString());
-
-      expect(result?.context).toBe("full content");
-      expect(result?.stageLogs).toEqual(stageLogs);
-    });
-
-    it("returns null when the scan does not match", async () => {
-      const fixture = createScanResourceFixture();
-      const { insertedId } = await getScanResourcesCollection().insertOne(fixture);
-
-      const result = await getScanResourceById(new ObjectId().toString(), insertedId.toString());
-
-      expect(result).toBeNull();
-    });
-  });
-});
+import { ObjectId } from "mongodb";
+import {
+  getScanResourcesCollection,
+  getScanKpiCounts,
+  getScanPages,
+  getScanResourceById,
+  getScanResourceCountsByScanIds,
+} from "../scanResources.getters";
+import { boardColumn } from "../scanResources.utils";
+import { createScanResourceFixture } from "../../../test-utils/factories";
+
+describe("db.websitalk.scanResources", () => {
+  describe("boardColumn()", () => {
+    it("folds URL_FAILED into FAILED, not READY_TO_PROCESS", () => {
+      expect(boardColumn("URL_FAILED")).toBe("FAILED");
+    });
+
+    it("folds URL_PERMANENT_FAILURE into FAILED", () => {
+      expect(boardColumn("URL_PERMANENT_FAILURE")).toBe("FAILED");
+    });
+
+    it("folds duplicate content into COMPLETED", () => {
+      expect(boardColumn("URL_DUPLICATE_CONTENT")).toBe("COMPLETED");
+    });
+  });
+
+  describe("getScanKpiCounts()", () => {
+    it("sums to the scan's total task count and omits zero-count statuses", async () => {
+      const scanId = new ObjectId().toString();
+      await getScanResourcesCollection().insertMany([
+        createScanResourceFixture({ scanId, status: "URL_COMPLETED" }),
+        createScanResourceFixture({ scanId, status: "URL_COMPLETED" }),
+        createScanResourceFixture({ scanId, status: "URL_PERMANENT_FAILURE" }),
+      ]);
+
+      const counts = await getScanKpiCounts(scanId);
+
+      expect(counts.URL_COMPLETED).toBe(2);
+      expect(counts.URL_PERMANENT_FAILURE).toBe(1);
+      expect(counts.URL_PENDING).toBeUndefined();
+      const total = Object.values(counts).reduce((sum, n) => sum + (n ?? 0), 0);
+      expect(total).toBe(3);
+    });
+  });
+
+  describe("getScanPages()", () => {
+    it("never returns context or stageLogs, and paginates within one column", async () => {
+      const scanId = new ObjectId().toString();
+      await getScanResourcesCollection().insertMany(
+        Array.from({ length: 3 }, () =>
+          createScanResourceFixture({
+            scanId,
+            status: "URL_COMPLETED",
+            context: "heavy markdown content",
+            stageLogs: [
+              {
+                from: null,
+                to: "URL_COMPLETED",
+                occurredAt: new Date(),
+                details: {},
+              },
+            ],
+          }),
+        ),
+      );
+
+      const page1 = await getScanPages({
+        scanId,
+        column: "COMPLETED",
+        limit: 2,
+      });
+      expect(page1.items).toHaveLength(2);
+      expect(page1.nextCursor).not.toBeNull();
+      page1.items.forEach((item) => {
+        expect(item).not.toHaveProperty("context");
+        expect(item).not.toHaveProperty("stageLogs");
+      });
+
+      const page2 = await getScanPages({
+        scanId,
+        column: "COMPLETED",
+        limit: 2,
+        cursor: page1.nextCursor,
+      });
+      expect(page2.items).toHaveLength(1);
+      expect(page2.nextCursor).toBeNull();
+    });
+
+    it("folds URL_FAILED into the FAILED column query", async () => {
+      const scanId = new ObjectId().toString();
+      await getScanResourcesCollection().insertMany([
+        createScanResourceFixture({ scanId, status: "URL_READY_TO_PROCESS" }),
+        createScanResourceFixture({ scanId, status: "URL_FAILED" }),
+        createScanResourceFixture({ scanId, status: "URL_PERMANENT_FAILURE" }),
+      ]);
+
+      const ready = await getScanPages({ scanId, column: "READY_TO_PROCESS" });
+      expect(ready.items).toHaveLength(1);
+
+      const failed = await getScanPages({ scanId, column: "FAILED" });
+      expect(failed.items).toHaveLength(2);
+    });
+
+    it("finds a URL containing regex-special characters when searched verbatim", async () => {
+      const scanId = new ObjectId().toString();
+      const url =
+        "https://www.industry.org.il/index.php?dir=site&page=expert&op=item&cs=3022";
+      await getScanResourcesCollection().insertMany([
+        createScanResourceFixture({ scanId, status: "URL_COMPLETED", url }),
+        createScanResourceFixture({
+          scanId,
+          status: "URL_COMPLETED",
+          url: "https://example.com/other",
+        }),
+      ]);
+
+      const result = await getScanPages({
+        scanId,
+        column: "COMPLETED",
+        search: url,
+      });
+
+      expect(result.items).toHaveLength(1);
+      expect(result.items[0].url).toBe(url);
+    });
+  });
+
+  describe("getScanResourceCountsByScanIds()", () => {
+    it("returns per-scan totals and omits scans with no resources", async () => {
+      const scanA = new ObjectId().toString();
+      const scanB = new ObjectId().toString();
+      await getScanResourcesCollection().insertMany([
+        createScanResourceFixture({ scanId: scanA, status: "URL_COMPLETED" }),
+        createScanResourceFixture({ scanId: scanA, status: "URL_COMPLETED" }),
+        createScanResourceFixture({ scanId: scanB, status: "URL_PENDING" }),
+      ]);
+
+      const counts = await getScanResourceCountsByScanIds([
+        scanA,
+        scanB,
+        "missing",
+      ]);
+      expect(counts[scanA]).toBe(2);
+      expect(counts[scanB]).toBe(1);
+      expect(counts.missing).toBeUndefined();
+    });
+  });
+
+  describe("getScanResourceById()", () => {
+    it("returns the full document including stageLogs in insertion order", async () => {
+      const scanId = new ObjectId().toString();
+      const stageLogs = [
+        {
+          from: null,
+          to: "URL_PENDING" as const,
+          occurredAt: new Date("2024-01-01"),
+          details: {},
+        },
+        {
+          from: "URL_PENDING" as const,
+          to: "URL_SCANNING" as const,
+          occurredAt: new Date("2024-01-02"),
+          details: {},
+        },
+      ];
+      const fixture = createScanResourceFixture({
+        scanId,
+        stageLogs,
+        context: "full content",
+      });
+      const { insertedId } =
+        await getScanResourcesCollection().insertOne(fixture);
+
+      const result = await getScanResourceById(scanId, insertedId.toString());
+
+      expect(result?.context).toBe("full content");
+      expect(result?.stageLogs).toEqual(stageLogs);
+    });
+
+    it("returns null when the scan does not match", async () => {
+      const fixture = createScanResourceFixture();
+      const { insertedId } =
+        await getScanResourcesCollection().insertOne(fixture);
+
+      const result = await getScanResourceById(
+        new ObjectId().toString(),
+        insertedId.toString(),
+      );
+
+      expect(result).toBeNull();
+    });
+  });
+});

package/src/websitalk/scanResources/index.ts

@@ -1,4 +1,4 @@
-export * from "./scanResources.constants";
-export * from "./scanResources.getters";
-export * from "./scanResources.types";
-export * from "./scanResources.utils";
+export * from "./scanResources.constants";
+export * from "./scanResources.getters";
+export * from "./scanResources.types";
+export * from "./scanResources.utils";