@talkpilot/core-db 1.3.44 → 1.3.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +49 -0
- package/README.md +41 -0
- package/dist/configuration/models/models.constants.d.ts +1 -1
- package/dist/configuration/models/models.constants.d.ts.map +1 -1
- package/dist/configuration/models/models.constants.js +1 -1
- package/dist/configuration/models/models.constants.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/municipal/departmentsSubjects/departmentsSubjects.setters.d.ts +10 -0
- package/dist/municipal/departmentsSubjects/departmentsSubjects.setters.d.ts.map +1 -0
- package/dist/municipal/departmentsSubjects/departmentsSubjects.setters.js +15 -0
- package/dist/municipal/departmentsSubjects/departmentsSubjects.setters.js.map +1 -0
- package/dist/municipal/streets/streets.setters.d.ts +10 -0
- package/dist/municipal/streets/streets.setters.d.ts.map +1 -0
- package/dist/municipal/streets/streets.setters.js +15 -0
- package/dist/municipal/streets/streets.setters.js.map +1 -0
- package/dist/reconcile/index.d.ts +5 -0
- package/dist/reconcile/index.d.ts.map +1 -0
- package/dist/reconcile/index.js +21 -0
- package/dist/reconcile/index.js.map +1 -0
- package/dist/reconcile/reconcile.diff.d.ts +4 -0
- package/dist/reconcile/reconcile.diff.d.ts.map +1 -0
- package/dist/reconcile/reconcile.diff.js +36 -0
- package/dist/reconcile/reconcile.diff.js.map +1 -0
- package/dist/reconcile/reconcile.execute.d.ts +7 -0
- package/dist/reconcile/reconcile.execute.d.ts.map +1 -0
- package/dist/reconcile/reconcile.execute.js +20 -0
- package/dist/reconcile/reconcile.execute.js.map +1 -0
- package/dist/reconcile/reconcile.ops.d.ts +12 -0
- package/dist/reconcile/reconcile.ops.d.ts.map +1 -0
- package/dist/reconcile/reconcile.ops.js +24 -0
- package/dist/reconcile/reconcile.ops.js.map +1 -0
- package/dist/reconcile/reconcile.types.d.ts +32 -0
- package/dist/reconcile/reconcile.types.d.ts.map +1 -0
- package/dist/reconcile/reconcile.types.js +3 -0
- package/dist/reconcile/reconcile.types.js.map +1 -0
- package/dist/talkpilot/calls/calls.types.d.ts +15 -0
- package/dist/talkpilot/calls/calls.types.d.ts.map +1 -1
- package/dist/talkpilot/clientsConfig/clientsConfig.getters.d.ts +39 -0
- package/dist/talkpilot/clientsConfig/clientsConfig.getters.d.ts.map +1 -1
- package/dist/talkpilot/clientsConfig/clientsConfig.getters.js +92 -1
- package/dist/talkpilot/clientsConfig/clientsConfig.getters.js.map +1 -1
- package/dist/talkpilot/clientsConfig/clientsConfig.types.d.ts +4 -0
- package/dist/talkpilot/clientsConfig/clientsConfig.types.d.ts.map +1 -1
- package/dist/talkpilot/clientsConfig/index.d.ts +1 -0
- package/dist/talkpilot/clientsConfig/index.d.ts.map +1 -1
- package/dist/talkpilot/clientsConfig/index.js +1 -0
- package/dist/talkpilot/clientsConfig/index.js.map +1 -1
- package/dist/test-utils/factories/index.d.ts +1 -0
- package/dist/test-utils/factories/index.d.ts.map +1 -1
- package/dist/test-utils/factories/index.js +1 -0
- package/dist/test-utils/factories/index.js.map +1 -1
- package/dist/test-utils/factories/websitalk/scans.d.ts.map +1 -1
- package/dist/test-utils/factories/websitalk/scans.js +5 -0
- package/dist/test-utils/factories/websitalk/scans.js.map +1 -1
- package/dist/websitalk/index.d.ts +1 -0
- package/dist/websitalk/index.d.ts.map +1 -1
- package/dist/websitalk/index.js +1 -0
- package/dist/websitalk/index.js.map +1 -1
- package/dist/websitalk/knowledgeDocuments/knowledgeDocuments.types.d.ts +2 -4
- package/dist/websitalk/knowledgeDocuments/knowledgeDocuments.types.d.ts.map +1 -1
- package/dist/websitalk/scans/scans.getters.d.ts.map +1 -1
- package/dist/websitalk/scans/scans.getters.js +8 -2
- package/dist/websitalk/scans/scans.getters.js.map +1 -1
- package/dist/websitalk/scans/scans.types.d.ts +5 -0
- package/dist/websitalk/scans/scans.types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/configuration/models/models.constants.ts +1 -1
- package/src/index.ts +1 -0
- package/src/talkpilot/calls/calls.types.ts +16 -0
- package/src/talkpilot/clientsConfig/__tests__/clientsConfig.getters.spec.ts +318 -0
- package/src/talkpilot/clientsConfig/clientsConfig.constants.ts +2 -0
- package/src/talkpilot/clientsConfig/clientsConfig.getters.ts +116 -2
- package/src/talkpilot/clientsConfig/clientsConfig.types.ts +4 -0
- package/src/talkpilot/clientsConfig/index.ts +1 -0
- package/src/test-utils/factories/index.ts +1 -0
- package/src/test-utils/factories/websitalk/scanResources.ts +26 -0
- package/src/test-utils/factories/websitalk/scans.ts +5 -0
- package/src/websitalk/index.ts +1 -0
- package/src/websitalk/knowledgeDocuments/__tests__/knowledgeDocuments.spec.ts +0 -19
- package/src/websitalk/knowledgeDocuments/knowledgeDocuments.types.ts +2 -2
- package/src/websitalk/scanResources/__tests__/scanResources.spec.ts +136 -0
- package/src/websitalk/scanResources/index.ts +4 -0
- package/src/websitalk/scanResources/scanResources.constants.ts +47 -0
- package/src/websitalk/scanResources/scanResources.getters.ts +94 -0
- package/src/websitalk/scanResources/scanResources.types.ts +90 -0
- package/src/websitalk/scanResources/scanResources.utils.ts +28 -0
- package/src/websitalk/scans/scans.getters.ts +8 -2
- package/src/websitalk/scans/scans.types.ts +9 -0
- package/dist/talkpilot/clients/clients.quota.constants.d.ts +0 -2
- package/dist/talkpilot/clients/clients.quota.constants.d.ts.map +0 -1
- package/dist/talkpilot/clients/clients.quota.constants.js +0 -5
- package/dist/talkpilot/clients/clients.quota.constants.js.map +0 -1
- package/dist/talkpilot/clients/clients.quota.utils.d.ts +0 -8
- package/dist/talkpilot/clients/clients.quota.utils.d.ts.map +0 -1
- package/dist/talkpilot/clients/clients.quota.utils.js +0 -45
- package/dist/talkpilot/clients/clients.quota.utils.js.map +0 -1
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { ObjectId } from "mongodb";
|
|
2
|
+
import {
|
|
3
|
+
getScanResourcesCollection,
|
|
4
|
+
getScanKpiCounts,
|
|
5
|
+
getScanPages,
|
|
6
|
+
getScanResourceById,
|
|
7
|
+
getScanResourceCountsByScanIds,
|
|
8
|
+
} from "../scanResources.getters";
|
|
9
|
+
import { boardColumn } from "../scanResources.utils";
|
|
10
|
+
import { createScanResourceFixture } from "../../../test-utils/factories";
|
|
11
|
+
|
|
12
|
+
describe("db.websitalk.scanResources", () => {
|
|
13
|
+
describe("boardColumn()", () => {
|
|
14
|
+
it("folds URL_FAILED into FAILED, not READY_TO_PROCESS", () => {
|
|
15
|
+
expect(boardColumn("URL_FAILED")).toBe("FAILED");
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
it("folds URL_PERMANENT_FAILURE into FAILED", () => {
|
|
19
|
+
expect(boardColumn("URL_PERMANENT_FAILURE")).toBe("FAILED");
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
it("folds duplicate content into COMPLETED", () => {
|
|
23
|
+
expect(boardColumn("URL_DUPLICATE_CONTENT")).toBe("COMPLETED");
|
|
24
|
+
});
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
describe("getScanKpiCounts()", () => {
|
|
28
|
+
it("sums to the scan's total task count and omits zero-count statuses", async () => {
|
|
29
|
+
const scanId = new ObjectId().toString();
|
|
30
|
+
await getScanResourcesCollection().insertMany([
|
|
31
|
+
createScanResourceFixture({ scanId, status: "URL_COMPLETED" }),
|
|
32
|
+
createScanResourceFixture({ scanId, status: "URL_COMPLETED" }),
|
|
33
|
+
createScanResourceFixture({ scanId, status: "URL_PERMANENT_FAILURE" }),
|
|
34
|
+
]);
|
|
35
|
+
|
|
36
|
+
const counts = await getScanKpiCounts(scanId);
|
|
37
|
+
|
|
38
|
+
expect(counts.URL_COMPLETED).toBe(2);
|
|
39
|
+
expect(counts.URL_PERMANENT_FAILURE).toBe(1);
|
|
40
|
+
expect(counts.URL_PENDING).toBeUndefined();
|
|
41
|
+
const total = Object.values(counts).reduce((sum, n) => sum + (n ?? 0), 0);
|
|
42
|
+
expect(total).toBe(3);
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
describe("getScanPages()", () => {
|
|
47
|
+
it("never returns context or stageLogs, and paginates within one column", async () => {
|
|
48
|
+
const scanId = new ObjectId().toString();
|
|
49
|
+
await getScanResourcesCollection().insertMany(
|
|
50
|
+
Array.from({ length: 3 }, () =>
|
|
51
|
+
createScanResourceFixture({
|
|
52
|
+
scanId,
|
|
53
|
+
status: "URL_COMPLETED",
|
|
54
|
+
context: "heavy markdown content",
|
|
55
|
+
stageLogs: [{ from: null, to: "URL_COMPLETED", occurredAt: new Date(), details: {} }],
|
|
56
|
+
}),
|
|
57
|
+
),
|
|
58
|
+
);
|
|
59
|
+
|
|
60
|
+
const page1 = await getScanPages({ scanId, column: "COMPLETED", limit: 2 });
|
|
61
|
+
expect(page1.items).toHaveLength(2);
|
|
62
|
+
expect(page1.nextCursor).not.toBeNull();
|
|
63
|
+
page1.items.forEach((item) => {
|
|
64
|
+
expect(item).not.toHaveProperty("context");
|
|
65
|
+
expect(item).not.toHaveProperty("stageLogs");
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
const page2 = await getScanPages({
|
|
69
|
+
scanId,
|
|
70
|
+
column: "COMPLETED",
|
|
71
|
+
limit: 2,
|
|
72
|
+
cursor: page1.nextCursor,
|
|
73
|
+
});
|
|
74
|
+
expect(page2.items).toHaveLength(1);
|
|
75
|
+
expect(page2.nextCursor).toBeNull();
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("folds URL_FAILED into the FAILED column query", async () => {
|
|
79
|
+
const scanId = new ObjectId().toString();
|
|
80
|
+
await getScanResourcesCollection().insertMany([
|
|
81
|
+
createScanResourceFixture({ scanId, status: "URL_READY_TO_PROCESS" }),
|
|
82
|
+
createScanResourceFixture({ scanId, status: "URL_FAILED" }),
|
|
83
|
+
createScanResourceFixture({ scanId, status: "URL_PERMANENT_FAILURE" }),
|
|
84
|
+
]);
|
|
85
|
+
|
|
86
|
+
const ready = await getScanPages({ scanId, column: "READY_TO_PROCESS" });
|
|
87
|
+
expect(ready.items).toHaveLength(1);
|
|
88
|
+
|
|
89
|
+
const failed = await getScanPages({ scanId, column: "FAILED" });
|
|
90
|
+
expect(failed.items).toHaveLength(2);
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
describe("getScanResourceCountsByScanIds()", () => {
|
|
95
|
+
it("returns per-scan totals and omits scans with no resources", async () => {
|
|
96
|
+
const scanA = new ObjectId().toString();
|
|
97
|
+
const scanB = new ObjectId().toString();
|
|
98
|
+
await getScanResourcesCollection().insertMany([
|
|
99
|
+
createScanResourceFixture({ scanId: scanA, status: "URL_COMPLETED" }),
|
|
100
|
+
createScanResourceFixture({ scanId: scanA, status: "URL_COMPLETED" }),
|
|
101
|
+
createScanResourceFixture({ scanId: scanB, status: "URL_PENDING" }),
|
|
102
|
+
]);
|
|
103
|
+
|
|
104
|
+
const counts = await getScanResourceCountsByScanIds([scanA, scanB, "missing"]);
|
|
105
|
+
expect(counts[scanA]).toBe(2);
|
|
106
|
+
expect(counts[scanB]).toBe(1);
|
|
107
|
+
expect(counts.missing).toBeUndefined();
|
|
108
|
+
});
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
describe("getScanResourceById()", () => {
|
|
112
|
+
it("returns the full document including stageLogs in insertion order", async () => {
|
|
113
|
+
const scanId = new ObjectId().toString();
|
|
114
|
+
const stageLogs = [
|
|
115
|
+
{ from: null, to: "URL_PENDING" as const, occurredAt: new Date("2024-01-01"), details: {} },
|
|
116
|
+
{ from: "URL_PENDING" as const, to: "URL_SCANNING" as const, occurredAt: new Date("2024-01-02"), details: {} },
|
|
117
|
+
];
|
|
118
|
+
const fixture = createScanResourceFixture({ scanId, stageLogs, context: "full content" });
|
|
119
|
+
const { insertedId } = await getScanResourcesCollection().insertOne(fixture);
|
|
120
|
+
|
|
121
|
+
const result = await getScanResourceById(scanId, insertedId.toString());
|
|
122
|
+
|
|
123
|
+
expect(result?.context).toBe("full content");
|
|
124
|
+
expect(result?.stageLogs).toEqual(stageLogs);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it("returns null when the scan does not match", async () => {
|
|
128
|
+
const fixture = createScanResourceFixture();
|
|
129
|
+
const { insertedId } = await getScanResourcesCollection().insertOne(fixture);
|
|
130
|
+
|
|
131
|
+
const result = await getScanResourceById(new ObjectId().toString(), insertedId.toString());
|
|
132
|
+
|
|
133
|
+
expect(result).toBeNull();
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
});
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export const SCAN_RESOURCES_COLLECTION = "scanResources";
|
|
2
|
+
|
|
3
|
+
export const URL_TASK_STATUSES = {
|
|
4
|
+
URL_PENDING: "URL_PENDING",
|
|
5
|
+
URL_SCANNING: "URL_SCANNING",
|
|
6
|
+
URL_READY_TO_PROCESS: "URL_READY_TO_PROCESS",
|
|
7
|
+
URL_PENDING_CHUNKS: "URL_PENDING_CHUNKS",
|
|
8
|
+
URL_CHUNKING: "URL_CHUNKING",
|
|
9
|
+
URL_EMBEDDING: "URL_EMBEDDING",
|
|
10
|
+
URL_COMPLETED: "URL_COMPLETED",
|
|
11
|
+
URL_FAILED: "URL_FAILED",
|
|
12
|
+
URL_DUPLICATE_CONTENT: "URL_DUPLICATE_CONTENT",
|
|
13
|
+
URL_PERMANENT_FAILURE: "URL_PERMANENT_FAILURE",
|
|
14
|
+
} as const;
|
|
15
|
+
|
|
16
|
+
export const BOARD_COLUMN_STATUSES = {
|
|
17
|
+
PENDING: [URL_TASK_STATUSES.URL_PENDING],
|
|
18
|
+
SCANNING: [URL_TASK_STATUSES.URL_SCANNING],
|
|
19
|
+
READY_TO_PROCESS: [URL_TASK_STATUSES.URL_READY_TO_PROCESS],
|
|
20
|
+
CHUNKING: [URL_TASK_STATUSES.URL_CHUNKING, URL_TASK_STATUSES.URL_PENDING_CHUNKS],
|
|
21
|
+
PROCESSING: [URL_TASK_STATUSES.URL_EMBEDDING],
|
|
22
|
+
COMPLETED: [
|
|
23
|
+
URL_TASK_STATUSES.URL_COMPLETED,
|
|
24
|
+
URL_TASK_STATUSES.URL_DUPLICATE_CONTENT,
|
|
25
|
+
],
|
|
26
|
+
FAILED: [
|
|
27
|
+
URL_TASK_STATUSES.URL_PERMANENT_FAILURE,
|
|
28
|
+
URL_TASK_STATUSES.URL_FAILED,
|
|
29
|
+
],
|
|
30
|
+
} as const;
|
|
31
|
+
|
|
32
|
+
export const SCAN_RESOURCE_BOARD_PROJECTION = {
|
|
33
|
+
url: 1,
|
|
34
|
+
title: 1,
|
|
35
|
+
status: 1,
|
|
36
|
+
statusEnteredAt: 1,
|
|
37
|
+
retryCount: 1,
|
|
38
|
+
processRetryCount: 1,
|
|
39
|
+
globalResurrections: 1,
|
|
40
|
+
noRetry: 1,
|
|
41
|
+
errorMessage: 1,
|
|
42
|
+
updatedAt: 1,
|
|
43
|
+
sourceType: 1,
|
|
44
|
+
} as const;
|
|
45
|
+
|
|
46
|
+
export const DEFAULT_PAGE_LIMIT = 4;
|
|
47
|
+
export const MAX_PAGE_LIMIT = 50;
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { Collection, Filter, ObjectId } from "mongodb";
|
|
2
|
+
import { getDb } from "../index";
|
|
3
|
+
import {
|
|
4
|
+
DEFAULT_PAGE_LIMIT,
|
|
5
|
+
MAX_PAGE_LIMIT,
|
|
6
|
+
SCAN_RESOURCES_COLLECTION,
|
|
7
|
+
SCAN_RESOURCE_BOARD_PROJECTION,
|
|
8
|
+
} from "./scanResources.constants";
|
|
9
|
+
import type {
|
|
10
|
+
PagedResult,
|
|
11
|
+
ScanPagesQuery,
|
|
12
|
+
ScanResource,
|
|
13
|
+
ScanResourceBoardProjection,
|
|
14
|
+
ScanResourceDoc,
|
|
15
|
+
UrlTaskStatus,
|
|
16
|
+
} from "./scanResources.types";
|
|
17
|
+
import { decodeCursor, encodeCursor, statusesForColumn } from "./scanResources.utils";
|
|
18
|
+
|
|
19
|
+
export const getScanResourcesCollection = (): Collection<ScanResource> =>
|
|
20
|
+
getDb().collection<ScanResource>(SCAN_RESOURCES_COLLECTION);
|
|
21
|
+
|
|
22
|
+
export const getScanKpiCounts = async (
|
|
23
|
+
scanId: string,
|
|
24
|
+
): Promise<Partial<Record<UrlTaskStatus, number>>> => {
|
|
25
|
+
const rows = await getScanResourcesCollection()
|
|
26
|
+
.aggregate<{ _id: UrlTaskStatus; count: number }>([
|
|
27
|
+
{ $match: { scanId } },
|
|
28
|
+
{ $group: { _id: "$status", count: { $sum: 1 } } },
|
|
29
|
+
])
|
|
30
|
+
.toArray();
|
|
31
|
+
|
|
32
|
+
return rows.reduce<Partial<Record<UrlTaskStatus, number>>>((acc, row) => {
|
|
33
|
+
acc[row._id] = row.count;
|
|
34
|
+
return acc;
|
|
35
|
+
}, {});
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
export const getScanResourceCountsByScanIds = async (
|
|
39
|
+
scanIds: string[],
|
|
40
|
+
): Promise<Record<string, number>> => {
|
|
41
|
+
if (scanIds.length === 0) return {};
|
|
42
|
+
|
|
43
|
+
const rows = await getScanResourcesCollection()
|
|
44
|
+
.aggregate<{ _id: string; count: number }>([
|
|
45
|
+
{ $match: { scanId: { $in: scanIds } } },
|
|
46
|
+
{ $group: { _id: "$scanId", count: { $sum: 1 } } },
|
|
47
|
+
])
|
|
48
|
+
.toArray();
|
|
49
|
+
|
|
50
|
+
return Object.fromEntries(rows.map((row) => [row._id, row.count]));
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
export const getScanPages = async (
|
|
54
|
+
query: ScanPagesQuery,
|
|
55
|
+
): Promise<PagedResult<ScanResourceBoardProjection>> => {
|
|
56
|
+
const limit = Math.min(query.limit ?? DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT);
|
|
57
|
+
|
|
58
|
+
const filter: Filter<ScanResource> = {
|
|
59
|
+
scanId: query.scanId,
|
|
60
|
+
status: { $in: [...statusesForColumn(query.column)] },
|
|
61
|
+
};
|
|
62
|
+
if (query.cursor) {
|
|
63
|
+
filter._id = { $gt: decodeCursor(query.cursor) };
|
|
64
|
+
}
|
|
65
|
+
if (query.search) {
|
|
66
|
+
filter.$or = [
|
|
67
|
+
{ url: { $regex: query.search, $options: "i" } },
|
|
68
|
+
{ title: { $regex: query.search, $options: "i" } },
|
|
69
|
+
];
|
|
70
|
+
}
|
|
71
|
+
if (query.since) {
|
|
72
|
+
filter.updatedAt = { $gte: query.since };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const docs = await getScanResourcesCollection()
|
|
76
|
+
.find(filter, {
|
|
77
|
+
projection: SCAN_RESOURCE_BOARD_PROJECTION,
|
|
78
|
+
sort: { _id: 1 },
|
|
79
|
+
limit: limit + 1,
|
|
80
|
+
})
|
|
81
|
+
.toArray();
|
|
82
|
+
|
|
83
|
+
const hasMore = docs.length > limit;
|
|
84
|
+
const items = (hasMore ? docs.slice(0, limit) : docs) as ScanResourceBoardProjection[];
|
|
85
|
+
const nextCursor = hasMore ? encodeCursor(items[items.length - 1]._id) : null;
|
|
86
|
+
|
|
87
|
+
return { items, nextCursor };
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
export const getScanResourceById = async (
|
|
91
|
+
scanId: string,
|
|
92
|
+
pageId: string,
|
|
93
|
+
): Promise<ScanResourceDoc | null> =>
|
|
94
|
+
getScanResourcesCollection().findOne({ scanId, _id: new ObjectId(pageId) });
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { WithId } from "mongodb";
|
|
2
|
+
import type {
|
|
3
|
+
BOARD_COLUMN_STATUSES,
|
|
4
|
+
URL_TASK_STATUSES,
|
|
5
|
+
} from "./scanResources.constants";
|
|
6
|
+
|
|
7
|
+
export type UrlTaskStatus =
|
|
8
|
+
(typeof URL_TASK_STATUSES)[keyof typeof URL_TASK_STATUSES];
|
|
9
|
+
|
|
10
|
+
export type BoardColumn = keyof typeof BOARD_COLUMN_STATUSES;
|
|
11
|
+
|
|
12
|
+
export type StageLogEntry = {
|
|
13
|
+
from: UrlTaskStatus | null;
|
|
14
|
+
to: UrlTaskStatus;
|
|
15
|
+
occurredAt: Date;
|
|
16
|
+
details: Record<string, unknown>;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
export type ScanResource = {
|
|
20
|
+
urlHashId: string;
|
|
21
|
+
clientId: string;
|
|
22
|
+
scanId: string;
|
|
23
|
+
websiteUrlId: string;
|
|
24
|
+
url: string;
|
|
25
|
+
urlCanonical?: string | null;
|
|
26
|
+
title?: string | null;
|
|
27
|
+
h1?: string | null;
|
|
28
|
+
language?: string | null;
|
|
29
|
+
sourceType?: string | null;
|
|
30
|
+
context?: string | null;
|
|
31
|
+
contentHash?: string | null;
|
|
32
|
+
duplicateOfTaskId?: string | null;
|
|
33
|
+
breadcrumbs?: unknown;
|
|
34
|
+
description?: string | null;
|
|
35
|
+
sourceUrl?: string | null;
|
|
36
|
+
errorMessage?: string | null;
|
|
37
|
+
|
|
38
|
+
status: UrlTaskStatus;
|
|
39
|
+
depth: number;
|
|
40
|
+
retryCount: number;
|
|
41
|
+
|
|
42
|
+
processRetryCount?: number;
|
|
43
|
+
globalResurrections?: number;
|
|
44
|
+
noRetry?: boolean;
|
|
45
|
+
reviveAt?: Date | null;
|
|
46
|
+
statusEnteredAt?: Date;
|
|
47
|
+
|
|
48
|
+
httpStatus?: number;
|
|
49
|
+
finalUrl?: string;
|
|
50
|
+
batchDone?: number;
|
|
51
|
+
batchTotal?: number;
|
|
52
|
+
|
|
53
|
+
stageLogs?: StageLogEntry[];
|
|
54
|
+
|
|
55
|
+
createdAt: Date;
|
|
56
|
+
updatedAt: Date;
|
|
57
|
+
fetchedAt?: Date | null;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
export type ScanResourceDoc = WithId<ScanResource>;
|
|
61
|
+
|
|
62
|
+
export type ScanResourceBoardProjection = Pick<
|
|
63
|
+
ScanResourceDoc,
|
|
64
|
+
| "_id"
|
|
65
|
+
| "url"
|
|
66
|
+
| "title"
|
|
67
|
+
| "status"
|
|
68
|
+
| "statusEnteredAt"
|
|
69
|
+
| "retryCount"
|
|
70
|
+
| "processRetryCount"
|
|
71
|
+
| "globalResurrections"
|
|
72
|
+
| "noRetry"
|
|
73
|
+
| "errorMessage"
|
|
74
|
+
| "updatedAt"
|
|
75
|
+
| "sourceType"
|
|
76
|
+
>;
|
|
77
|
+
|
|
78
|
+
export type ScanPagesQuery = {
|
|
79
|
+
scanId: string;
|
|
80
|
+
column: BoardColumn;
|
|
81
|
+
search?: string;
|
|
82
|
+
cursor?: string | null;
|
|
83
|
+
since?: Date;
|
|
84
|
+
limit?: number;
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
export type PagedResult<T> = {
|
|
88
|
+
items: T[];
|
|
89
|
+
nextCursor: string | null;
|
|
90
|
+
};
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { ObjectId } from "mongodb";
|
|
2
|
+
import { BOARD_COLUMN_STATUSES } from "./scanResources.constants";
|
|
3
|
+
import type { BoardColumn, UrlTaskStatus } from "./scanResources.types";
|
|
4
|
+
|
|
5
|
+
const STATUS_TO_COLUMN: Record<UrlTaskStatus, BoardColumn> = Object.entries(
|
|
6
|
+
BOARD_COLUMN_STATUSES,
|
|
7
|
+
).reduce(
|
|
8
|
+
(acc, [column, statuses]) => {
|
|
9
|
+
for (const status of statuses) {
|
|
10
|
+
acc[status] = column as BoardColumn;
|
|
11
|
+
}
|
|
12
|
+
return acc;
|
|
13
|
+
},
|
|
14
|
+
{} as Record<UrlTaskStatus, BoardColumn>,
|
|
15
|
+
);
|
|
16
|
+
|
|
17
|
+
export const boardColumn = (status: UrlTaskStatus): BoardColumn =>
|
|
18
|
+
STATUS_TO_COLUMN[status];
|
|
19
|
+
|
|
20
|
+
export const statusesForColumn = (
|
|
21
|
+
column: BoardColumn,
|
|
22
|
+
): readonly UrlTaskStatus[] => BOARD_COLUMN_STATUSES[column];
|
|
23
|
+
|
|
24
|
+
export const encodeCursor = (id: ObjectId): string =>
|
|
25
|
+
Buffer.from(id.toHexString()).toString("base64");
|
|
26
|
+
|
|
27
|
+
export const decodeCursor = (cursor: string): ObjectId =>
|
|
28
|
+
new ObjectId(Buffer.from(cursor, "base64").toString("utf-8"));
|
|
@@ -57,6 +57,7 @@ export const createScanDoc = (
|
|
|
57
57
|
scan: Pick<Scan, "clientId" | "websiteUrlId"> &
|
|
58
58
|
Partial<Pick<Scan, "notifyPhoneNumber">>,
|
|
59
59
|
) => {
|
|
60
|
+
const now = new Date();
|
|
60
61
|
return getScansCollection().insertOne({
|
|
61
62
|
...scan,
|
|
62
63
|
status: SCAN_STATUSES.CHECKING,
|
|
@@ -65,8 +66,13 @@ export const createScanDoc = (
|
|
|
65
66
|
pagesSkipped: 0,
|
|
66
67
|
skipped: [],
|
|
67
68
|
siteChrome: {} as Scan["siteChrome"],
|
|
68
|
-
createdAt:
|
|
69
|
-
updatedAt:
|
|
69
|
+
createdAt: now,
|
|
70
|
+
updatedAt: now,
|
|
71
|
+
startedAt: now,
|
|
72
|
+
completedAt: null,
|
|
73
|
+
pagesCompleted: 0,
|
|
74
|
+
pagesFailedPermanent: 0,
|
|
75
|
+
pagesDuplicate: 0,
|
|
70
76
|
});
|
|
71
77
|
};
|
|
72
78
|
|
|
@@ -40,6 +40,15 @@ export type Scan = {
|
|
|
40
40
|
siteChrome: SiteChromeEntry;
|
|
41
41
|
createdAt: Date;
|
|
42
42
|
updatedAt: Date;
|
|
43
|
+
|
|
44
|
+
// Terminal-only snapshot, written once at finalize (Brain, outside core-db).
|
|
45
|
+
// Never read for live progress — an active scan's KPIs must come from
|
|
46
|
+
// getScanKpiCounts, not these fields, which are not kept in sync mid-scan.
|
|
47
|
+
startedAt?: Date;
|
|
48
|
+
completedAt?: Date | null;
|
|
49
|
+
pagesCompleted?: number;
|
|
50
|
+
pagesFailedPermanent?: number;
|
|
51
|
+
pagesDuplicate?: number;
|
|
43
52
|
};
|
|
44
53
|
|
|
45
54
|
export type ScanDoc = WithId<Scan>;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"clients.quota.constants.d.ts","sourceRoot":"","sources":["../../../src/talkpilot/clients/clients.quota.constants.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,oBAAoB,MAAM,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"clients.quota.constants.js","sourceRoot":"","sources":["../../../src/talkpilot/clients/clients.quota.constants.ts"],"names":[],"mappings":";;;AAAa,QAAA,oBAAoB,GAAG,GAAG,CAAC"}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { ClientQuotaNotifications, ClientQuotaState, ClientQuotaStatus, QuotaThresholdAlert } from "./clients.quota.types";
|
|
2
|
-
export declare const computeClientQuotaStatus: (quota: ClientQuotaState | null) => ClientQuotaStatus;
|
|
3
|
-
export declare const getQuotaAlerts: (quota?: ClientQuotaState) => QuotaThresholdAlert[];
|
|
4
|
-
export declare const findFullQuotaAlertIndex: (alerts: QuotaThresholdAlert[]) => number;
|
|
5
|
-
export declare const getFullQuotaAlert: (quota?: ClientQuotaState) => QuotaThresholdAlert | undefined;
|
|
6
|
-
export declare const isFullQuotaAlert: (thresholdPct: number) => boolean;
|
|
7
|
-
export declare const toClientQuotaNotifications: (quota: ClientQuotaState | null) => ClientQuotaNotifications;
|
|
8
|
-
//# sourceMappingURL=clients.quota.utils.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"clients.quota.utils.d.ts","sourceRoot":"","sources":["../../../src/talkpilot/clients/clients.quota.utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,wBAAwB,EACxB,gBAAgB,EAChB,iBAAiB,EACjB,mBAAmB,EACpB,MAAM,uBAAuB,CAAC;AAQ/B,eAAO,MAAM,wBAAwB,GACnC,OAAO,gBAAgB,GAAG,IAAI,KAC7B,iBAiBF,CAAC;AAEF,eAAO,MAAM,cAAc,GAAI,QAAQ,gBAAgB,KAAG,mBAAmB,EAClC,CAAC;AAE5C,eAAO,MAAM,uBAAuB,GAAI,QAAQ,mBAAmB,EAAE,KAAG,MACP,CAAC;AAElE,eAAO,MAAM,iBAAiB,GAC5B,QAAQ,gBAAgB,KACvB,mBAAmB,GAAG,SAIxB,CAAC;AAEF,eAAO,MAAM,gBAAgB,GAAI,cAAc,MAAM,KAAG,OAClB,CAAC;AAEvC,eAAO,MAAM,0BAA0B,GACrC,OAAO,gBAAgB,GAAG,IAAI,KAC7B,wBASD,CAAC"}
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.toClientQuotaNotifications = exports.isFullQuotaAlert = exports.getFullQuotaAlert = exports.findFullQuotaAlertIndex = exports.getQuotaAlerts = exports.computeClientQuotaStatus = void 0;
|
|
4
|
-
const clients_quota_constants_1 = require("./clients.quota.constants");
|
|
5
|
-
const computeUsagePct = (usedQuota, totalQuota) => totalQuota > 0 ? Math.round((usedQuota / totalQuota) * 100) : 0;
|
|
6
|
-
const isPastExpiration = (expiresAt) => expiresAt ? Date.now() > new Date(expiresAt).getTime() : false;
|
|
7
|
-
const computeClientQuotaStatus = (quota) => {
|
|
8
|
-
if (!quota) {
|
|
9
|
-
return { hasQuota: false };
|
|
10
|
-
}
|
|
11
|
-
const totalQuota = quota.totalQuota;
|
|
12
|
-
const usedQuota = quota.usedQuota ?? 0;
|
|
13
|
-
return {
|
|
14
|
-
hasQuota: true,
|
|
15
|
-
totalQuota,
|
|
16
|
-
usedQuota,
|
|
17
|
-
pct: computeUsagePct(usedQuota, totalQuota),
|
|
18
|
-
expiresAt: quota.expiresAt ? new Date(quota.expiresAt).toISOString() : null,
|
|
19
|
-
isExceeded: usedQuota >= totalQuota,
|
|
20
|
-
isExpired: isPastExpiration(quota.expiresAt),
|
|
21
|
-
};
|
|
22
|
-
};
|
|
23
|
-
exports.computeClientQuotaStatus = computeClientQuotaStatus;
|
|
24
|
-
const getQuotaAlerts = (quota) => quota?.alerts?.length ? quota.alerts : [];
|
|
25
|
-
exports.getQuotaAlerts = getQuotaAlerts;
|
|
26
|
-
const findFullQuotaAlertIndex = (alerts) => alerts.findIndex((a) => a.thresholdPct >= clients_quota_constants_1.FULL_QUOTA_THRESHOLD);
|
|
27
|
-
exports.findFullQuotaAlertIndex = findFullQuotaAlertIndex;
|
|
28
|
-
const getFullQuotaAlert = (quota) => {
|
|
29
|
-
const alerts = (0, exports.getQuotaAlerts)(quota);
|
|
30
|
-
const index = (0, exports.findFullQuotaAlertIndex)(alerts);
|
|
31
|
-
return index >= 0 ? alerts[index] : undefined;
|
|
32
|
-
};
|
|
33
|
-
exports.getFullQuotaAlert = getFullQuotaAlert;
|
|
34
|
-
const isFullQuotaAlert = (thresholdPct) => thresholdPct >= clients_quota_constants_1.FULL_QUOTA_THRESHOLD;
|
|
35
|
-
exports.isFullQuotaAlert = isFullQuotaAlert;
|
|
36
|
-
const toClientQuotaNotifications = (quota) => ({
|
|
37
|
-
alerts: (0, exports.getQuotaAlerts)(quota ?? undefined).map(({ thresholdPct, recipientEmails, thresholdSent }) => ({
|
|
38
|
-
thresholdPct,
|
|
39
|
-
recipientEmails: recipientEmails ?? [],
|
|
40
|
-
...(thresholdSent ? { thresholdSent: true } : {}),
|
|
41
|
-
})),
|
|
42
|
-
...(quota?.expiredSent ? { expiredSent: true } : {}),
|
|
43
|
-
});
|
|
44
|
-
exports.toClientQuotaNotifications = toClientQuotaNotifications;
|
|
45
|
-
//# sourceMappingURL=clients.quota.utils.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"clients.quota.utils.js","sourceRoot":"","sources":["../../../src/talkpilot/clients/clients.quota.utils.ts"],"names":[],"mappings":";;;AAAA,uEAAiE;AAQjE,MAAM,eAAe,GAAG,CAAC,SAAiB,EAAE,UAAkB,EAAU,EAAE,CACxE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAElE,MAAM,gBAAgB,GAAG,CAAC,SAAwC,EAAW,EAAE,CAC7E,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;AAE1D,MAAM,wBAAwB,GAAG,CACtC,KAA8B,EACX,EAAE;IACrB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IAC7B,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;IACpC,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC;IAEvC,OAAO;QACL,QAAQ,EAAE,IAAI;QACd,UAAU;QACV,SAAS;QACT,GAAG,EAAE,eAAe,CAAC,SAAS,EAAE,UAAU,CAAC;QAC3C,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;QAC3E,UAAU,EAAE,SAAS,IAAI,UAAU;QACnC,SAAS,EAAE,gBAAgB,CAAC,KAAK,CAAC,SAAS,CAAC;KAC7C,CAAC;AACJ,CAAC,CAAC;AAnBW,QAAA,wBAAwB,4BAmBnC;AAEK,MAAM,cAAc,GAAG,CAAC,KAAwB,EAAyB,EAAE,CAChF,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;AAD/B,QAAA,cAAc,kBACiB;AAErC,MAAM,uBAAuB,GAAG,CAAC,MAA6B,EAAU,EAAE,CAC/E,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,IAAI,8CAAoB,CAAC,CAAC;AADrD,QAAA,uBAAuB,2BAC8B;AAE3D,MAAM,iBAAiB,GAAG,CAC/B,KAAwB,EACS,EAAE;IACnC,MAAM,MAAM,GAAG,IAAA,sBAAc,EAAC,KAAK,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,IAAA,+BAAuB,EAAC,MAAM,CAAC,CAAC;IAC9C,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAChD,CAAC,CAAC;AANW,QAAA,iBAAiB,qBAM5B;AAEK,MAAM,gBAAgB,GAAG,CAAC,YAAoB,EAAW,EAAE,CAChE,YAAY,IAAI,8CAAoB,CAAC;AAD1B,QAAA,gBAAgB,oBACU;AAEhC,MAAM,0BAA0B,GAAG,CACxC,KAA8B,EACJ,EAAE,CAAC,CAAC;IAC9B,MAAM,EAAE,IAAA,sBAAc,EAAC,KAAK,IAAI,SAAS,CAAC,CAAC,GAAG,CAC5C,CAAC,EAAE,YAAY,EAAE,eAAe,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;QACrD,YAAY;QACZ,eAAe,EAAE,eAAe,IAAI,EAAE;QACtC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,IAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3D,CAAC,CACH;IACD,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;CAC9D,CAAC,CAAC;AAXU,QAAA,0BAA0B,8BAWpC"}
|