@cleaver/breakcheck-core 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -9
- package/dist/api/index.d.ts +24 -0
- package/dist/api/index.js +6 -6
- package/dist/api/index.js.map +1 -1
- package/dist/core/compare/classes/ComparisonRepository.d.ts +25 -0
- package/dist/core/compare/classes/ComparisonRepository.js +1 -1
- package/dist/core/compare/classes/ComparisonRepository.js.map +1 -1
- package/dist/core/compare/index.d.ts +14 -0
- package/dist/core/compare/index.js +1 -1
- package/dist/core/compare/index.js.map +1 -1
- package/dist/core/crawler/implementations/cheerio.d.ts +3 -0
- package/dist/core/crawler/implementations/playwright.d.ts +3 -0
- package/dist/core/crawler/index.d.ts +10 -0
- package/dist/core/crawler/index.js +3 -3
- package/dist/core/crawler/index.js.map +1 -1
- package/dist/core/rules/RulesDsl.d.ts +2 -0
- package/dist/core/rules/RulesDsl.js +9 -12
- package/dist/core/rules/RulesDsl.js.map +1 -1
- package/dist/core/rules/RulesEngine.d.ts +8 -0
- package/dist/core/rules/RulesEngine.js +5 -19
- package/dist/core/rules/RulesEngine.js.map +1 -1
- package/dist/core/snapshot/classes/LoadedSnapshot.d.ts +12 -0
- package/dist/core/snapshot/classes/LoadedSnapshot.js +1 -1
- package/dist/core/snapshot/classes/LoadedSnapshot.js.map +1 -1
- package/dist/core/snapshot/classes/SnapshotRepository.d.ts +43 -0
- package/dist/core/snapshot/classes/SnapshotRepository.js +2 -2
- package/dist/core/snapshot/classes/SnapshotRepository.js.map +1 -1
- package/dist/core/snapshot/index.d.ts +9 -0
- package/dist/core/snapshot/index.js +4 -4
- package/dist/core/snapshot/index.js.map +1 -1
- package/dist/core/view/index.d.ts +2 -0
- package/dist/core/view/index.handlers.d.ts +3 -0
- package/dist/core/view/index.handlers.js +1 -1
- package/dist/core/view/index.handlers.js.map +1 -1
- package/dist/core/view/index.js +3 -16
- package/dist/core/view/index.js.map +1 -1
- package/dist/index.d.ts +13 -14
- package/dist/index.js +10 -11
- package/dist/index.js.map +1 -1
- package/dist/lib/logger.d.ts +8 -0
- package/dist/lib/logger.js +1 -1
- package/dist/lib/logger.js.map +1 -1
- package/dist/lib/root.d.ts +7 -0
- package/dist/types/api.d.ts +173 -0
- package/dist/types/compare.d.ts +55 -0
- package/dist/types/crawler.d.ts +20 -0
- package/dist/types/rules.d.ts +61 -0
- package/dist/types/snapshot.d.ts +31 -0
- package/package.json +7 -1
- package/dist/__tests__/compare/index.test.js +0 -322
- package/dist/__tests__/compare/index.test.js.map +0 -1
- package/dist/__tests__/rules/RulesDsl.test.js +0 -190
- package/dist/__tests__/rules/RulesDsl.test.js.map +0 -1
- package/dist/__tests__/rules/RulesEngine.test.js +0 -142
- package/dist/__tests__/rules/RulesEngine.test.js.map +0 -1
- package/dist/__tests__/snapshot/index.test.js +0 -238
- package/dist/__tests__/snapshot/index.test.js.map +0 -1
- package/dist/__tests__/view/index.test.js +0 -102
- package/dist/__tests__/view/index.test.js.map +0 -1
|
@@ -1,142 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { RulesEngine } from "../../core/rules/RulesEngine";
|
|
3
|
-
describe("RulesEngine", () => {
|
|
4
|
-
describe("create", () => {
|
|
5
|
-
it("should create instance with ruleset object", async () => {
|
|
6
|
-
const ruleset = {
|
|
7
|
-
name: "test-ruleset",
|
|
8
|
-
rules: [],
|
|
9
|
-
};
|
|
10
|
-
const engine = await RulesEngine.create(ruleset);
|
|
11
|
-
expect(engine).toBeInstanceOf(RulesEngine);
|
|
12
|
-
});
|
|
13
|
-
it("should create instance with ruleset name", async () => {
|
|
14
|
-
// This test would require a mock filesystem or actual rules file
|
|
15
|
-
// We'll skip this for now as it requires more setup
|
|
16
|
-
});
|
|
17
|
-
});
|
|
18
|
-
describe("process", () => {
|
|
19
|
-
it("should handle empty ruleset", async () => {
|
|
20
|
-
const ruleset = {
|
|
21
|
-
name: "empty-ruleset",
|
|
22
|
-
rules: [],
|
|
23
|
-
};
|
|
24
|
-
const engine = await RulesEngine.create(ruleset);
|
|
25
|
-
const html = "<div>Test</div>";
|
|
26
|
-
const expected = "<html><head></head><body><div>Test</div></body></html>";
|
|
27
|
-
expect(engine.process(html)).toBe(expected);
|
|
28
|
-
});
|
|
29
|
-
it("should exclude elements matching selector", async () => {
|
|
30
|
-
const ruleset = {
|
|
31
|
-
name: "exclude-ruleset",
|
|
32
|
-
rules: [
|
|
33
|
-
{
|
|
34
|
-
selector: ".ad",
|
|
35
|
-
actions: [{ action: "exclude" }],
|
|
36
|
-
},
|
|
37
|
-
],
|
|
38
|
-
};
|
|
39
|
-
const engine = await RulesEngine.create(ruleset);
|
|
40
|
-
const html = '<div class="ad">Ad content</div><div>Regular content</div>';
|
|
41
|
-
const expected = "<html><head></head><body><div>Regular content</div></body></html>";
|
|
42
|
-
expect(engine.process(html)).toBe(expected);
|
|
43
|
-
});
|
|
44
|
-
it("should remove attributes", async () => {
|
|
45
|
-
const ruleset = {
|
|
46
|
-
name: "remove-attr-ruleset",
|
|
47
|
-
rules: [
|
|
48
|
-
{
|
|
49
|
-
selector: "img",
|
|
50
|
-
actions: [
|
|
51
|
-
{
|
|
52
|
-
action: "remove_attr",
|
|
53
|
-
modifiers: { attr: "srcset" },
|
|
54
|
-
},
|
|
55
|
-
],
|
|
56
|
-
},
|
|
57
|
-
],
|
|
58
|
-
};
|
|
59
|
-
const engine = await RulesEngine.create(ruleset);
|
|
60
|
-
const html = '<img src="test.jpg" srcset="test.jpg 1x, test@2x.jpg 2x">';
|
|
61
|
-
const expected = '<html><head></head><body><img src="test.jpg"></body></html>';
|
|
62
|
-
expect(engine.process(html)).toBe(expected);
|
|
63
|
-
});
|
|
64
|
-
it("should rewrite attributes", async () => {
|
|
65
|
-
const ruleset = {
|
|
66
|
-
name: "rewrite-attr-ruleset",
|
|
67
|
-
rules: [
|
|
68
|
-
{
|
|
69
|
-
selector: "a",
|
|
70
|
-
actions: [
|
|
71
|
-
{
|
|
72
|
-
action: "rewrite_attr",
|
|
73
|
-
modifiers: {
|
|
74
|
-
attr: "href",
|
|
75
|
-
regex: "/user/\\d+",
|
|
76
|
-
replace: "/user/USER_ID",
|
|
77
|
-
},
|
|
78
|
-
},
|
|
79
|
-
],
|
|
80
|
-
},
|
|
81
|
-
],
|
|
82
|
-
};
|
|
83
|
-
const engine = await RulesEngine.create(ruleset);
|
|
84
|
-
const html = '<a href="/user/123">Profile</a>';
|
|
85
|
-
const expected = '<html><head></head><body><a href="/user/USER_ID">Profile</a></body></html>';
|
|
86
|
-
expect(engine.process(html)).toBe(expected);
|
|
87
|
-
});
|
|
88
|
-
it("should rewrite content", async () => {
|
|
89
|
-
const ruleset = {
|
|
90
|
-
name: "rewrite-content-ruleset",
|
|
91
|
-
rules: [
|
|
92
|
-
{
|
|
93
|
-
selector: ".timestamp",
|
|
94
|
-
actions: [
|
|
95
|
-
{
|
|
96
|
-
action: "rewrite_content",
|
|
97
|
-
modifiers: {
|
|
98
|
-
regex: "\\d{2}/\\d{2}/\\d{4}",
|
|
99
|
-
replace: "DATE_STAMP",
|
|
100
|
-
},
|
|
101
|
-
},
|
|
102
|
-
],
|
|
103
|
-
},
|
|
104
|
-
],
|
|
105
|
-
};
|
|
106
|
-
const engine = await RulesEngine.create(ruleset);
|
|
107
|
-
const html = '<div class="timestamp">Posted on 12/31/2023</div>';
|
|
108
|
-
const expected = '<html><head></head><body><div class="timestamp">Posted on DATE_STAMP</div></body></html>';
|
|
109
|
-
expect(engine.process(html)).toBe(expected);
|
|
110
|
-
});
|
|
111
|
-
it("should apply content regex filter", async () => {
|
|
112
|
-
const ruleset = {
|
|
113
|
-
name: "content-regex-ruleset",
|
|
114
|
-
rules: [
|
|
115
|
-
{
|
|
116
|
-
selector: ".message",
|
|
117
|
-
actions: [
|
|
118
|
-
{
|
|
119
|
-
action: "exclude",
|
|
120
|
-
modifiers: {
|
|
121
|
-
content_regex: "Logged in: \\d+ minutes ago",
|
|
122
|
-
},
|
|
123
|
-
},
|
|
124
|
-
],
|
|
125
|
-
},
|
|
126
|
-
],
|
|
127
|
-
};
|
|
128
|
-
const engine = await RulesEngine.create(ruleset);
|
|
129
|
-
const html = `
|
|
130
|
-
<div class="message">Logged in: 5 minutes ago</div>
|
|
131
|
-
<div class="message">Regular message</div>
|
|
132
|
-
`;
|
|
133
|
-
const expected = `
|
|
134
|
-
<html><head></head><body>
|
|
135
|
-
<div class="message">Regular message</div>
|
|
136
|
-
</body></html>
|
|
137
|
-
`;
|
|
138
|
-
expect(engine.process(html).replace(/\s+/g, " ").trim()).toBe(expected.replace(/\s+/g, " ").trim());
|
|
139
|
-
});
|
|
140
|
-
});
|
|
141
|
-
});
|
|
142
|
-
//# sourceMappingURL=RulesEngine.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"RulesEngine.test.js","sourceRoot":"","sources":["../../../src/__tests__/rules/RulesEngine.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAG3D,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;QACtB,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,OAAO,GAAY;gBACvB,IAAI,EAAE,cAAc;gBACpB,KAAK,EAAE,EAAE;aACV,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACjD,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,iEAAiE;YACjE,oDAAoD;QACtD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,OAAO,GAAY;gBACvB,IAAI,EAAE,eAAe;gBACrB,KAAK,EAAE,EAAE;aACV,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACjD,MAAM,IAAI,GAAG,iBAAiB,CAAC;YAC/B,MAAM,QAAQ,GAAG,wDAAwD,CAAC;YAC1E,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,OAAO,GAAY;gBACvB,IAAI,EAAE,iBAAiB;gBACvB,KAAK,EAAE;oBACL;wBACE,QAAQ,EAAE,KAAK;wBACf,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;qBACjC;iBACF;aACF,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACjD,MAAM,IAAI,GAAG,4DAA4D,CAAC;YAC1E,MAAM,QAAQ,GACZ,mEAAmE,CAAC;YACtE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YACxC,MAAM,OAAO,GAAY;gBACvB,IAAI,EAAE,qBAAqB;gBAC3B,KAAK,EAAE;oBACL;wBACE,QAAQ,EAAE,KAAK;wBACf,OAAO,EAAE;4BACP;gCACE,MAAM,EAAE,aAAa;gCACrB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;6BAC9B;yBACF;qBACF;iBACF;aACF,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACjD,MAAM,IAAI,GAAG,2DAA2D,CAAC;YACzE,MAAM,QAAQ,GACZ,6DAA6D,CAAC;YAChE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;YACzC,MAAM,OAAO,GAAY;gBACvB,IAAI,EAAE,sBAAsB;gBAC5B,KAAK,EAAE;oBACL;wBACE,QAAQ,EAAE,GAAG;wBACb,OAAO,EAAE;4BACP;gCACE,MAAM,EAAE,cAAc;gCACtB,SAAS,EAAE;oCACT,IAAI,EAAE,MAAM;oCACZ,KAAK,EAAE,YAAY;oCACnB,OAAO,EAAE,eAAe;iCACzB;6BACF;yBACF;qBACF;iBACF;aACF,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACjD,MAAM,IAAI,GAAG,iCAAiC,CAAC;YAC/C,MAAM,QAAQ,GACZ,4EAA4E,CAAC;YAC/E,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wBAAwB,EAAE,KAAK,IAAI,EAAE;YACtC,MAAM,OAAO,GAAY;gBACvB,IAAI,EAAE,yBAAyB;gBAC/B,KAAK,EAAE;oBACL;wBACE,QAAQ,EAAE,YAAY;wBACtB,OAAO,EAAE;4BACP;gCACE,MAAM,EAAE,iBAAiB;gCACzB,SAAS,EAAE;oCACT,KAAK,EAAE,sBAAsB;oCAC7B,OAAO,EAAE,YAAY;iCACtB;6BACF;yBACF;qBACF;iBACF;aACF,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACjD,MAAM,IAAI,GAAG,mDAAmD,CAAC;YACjE,MAAM,QAAQ,GACZ,0FAA0F,CAAC;YAC7F,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,OAAO,GAAY;gBACvB,IAAI,EAAE,uBAAuB;gBAC7B,KAAK,EAAE;oBACL;wBACE,QAAQ,EAAE,UAAU;wBACpB,OAAO,EAAE;4BACP;gCACE,MAAM,EAAE,SAAS;gCACjB,SAAS,EAAE;oCACT,aAAa,EAAE,6BAA6B;iCAC7C;6BACF;yBACF;qBACF;iBACF;aACF,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACjD,MAAM,IAAI,GAAG;;;OAGZ,CAAC;YACF,MAAM,QAAQ,GAAG;;;;OAIhB,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAC3D,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CACrC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1,238 +0,0 @@
|
|
|
1
|
-
import { Dataset, purgeDefaultStorages } from "crawlee";
|
|
2
|
-
import * as fs from "fs/promises";
|
|
3
|
-
import * as path from "path";
|
|
4
|
-
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
5
|
-
import { LoadedSnapshot, SnapshotRepository } from "../../core/snapshot";
|
|
6
|
-
describe("SnapshotRepository", () => {
|
|
7
|
-
const TEST_SNAPSHOTS_DIR = path.join(process.cwd(), "test-snapshots");
|
|
8
|
-
let snapshotRepository;
|
|
9
|
-
let testDataset;
|
|
10
|
-
beforeEach(async () => {
|
|
11
|
-
// Initialize storage system
|
|
12
|
-
await purgeDefaultStorages();
|
|
13
|
-
// Create a fresh test snapshots directory
|
|
14
|
-
await fs.mkdir(TEST_SNAPSHOTS_DIR, { recursive: true });
|
|
15
|
-
snapshotRepository = await SnapshotRepository.createWithCustomDir(TEST_SNAPSHOTS_DIR);
|
|
16
|
-
// Create a test dataset
|
|
17
|
-
testDataset = await Dataset.open("test-dataset");
|
|
18
|
-
// Create initial data
|
|
19
|
-
await testDataset.pushData({
|
|
20
|
-
type: "page",
|
|
21
|
-
url: "https://example.com",
|
|
22
|
-
finalUrl: "https://example.com",
|
|
23
|
-
statusCode: 200,
|
|
24
|
-
title: "Example Page",
|
|
25
|
-
html: "<html><body>Test</body></html>",
|
|
26
|
-
headers: { "content-type": "text/html" },
|
|
27
|
-
});
|
|
28
|
-
});
|
|
29
|
-
afterEach(async () => {
|
|
30
|
-
// Clean up test snapshots directory
|
|
31
|
-
await fs.rm(TEST_SNAPSHOTS_DIR, { recursive: true, force: true });
|
|
32
|
-
// Clean up dataset
|
|
33
|
-
await testDataset.drop();
|
|
34
|
-
});
|
|
35
|
-
describe("saveSnapshot", () => {
|
|
36
|
-
it("should save a snapshot with metadata and pages", async () => {
|
|
37
|
-
const metadata = {
|
|
38
|
-
baseUrl: "https://example.com",
|
|
39
|
-
timestamp: new Date().toISOString(),
|
|
40
|
-
crawlSettings: {
|
|
41
|
-
baseUrl: "https://example.com",
|
|
42
|
-
crawlerType: "cheerio",
|
|
43
|
-
maxRequestsPerCrawl: 10,
|
|
44
|
-
},
|
|
45
|
-
};
|
|
46
|
-
const pageCount = await snapshotRepository.saveSnapshot("test-snapshot", {
|
|
47
|
-
dataset: testDataset,
|
|
48
|
-
metadata,
|
|
49
|
-
});
|
|
50
|
-
expect(pageCount).toBe(1);
|
|
51
|
-
// Verify files were created
|
|
52
|
-
const snapshotDir = path.join(TEST_SNAPSHOTS_DIR, "test-snapshot");
|
|
53
|
-
const files = await fs.readdir(snapshotDir);
|
|
54
|
-
expect(files).toContain("metadata.json");
|
|
55
|
-
expect(files).toContain("index.json");
|
|
56
|
-
expect(files).toContain("pages");
|
|
57
|
-
// Verify metadata
|
|
58
|
-
const metadataContent = await fs.readFile(path.join(snapshotDir, "metadata.json"), "utf-8");
|
|
59
|
-
const savedMetadata = JSON.parse(metadataContent);
|
|
60
|
-
expect(savedMetadata).toEqual(metadata);
|
|
61
|
-
// Verify index
|
|
62
|
-
const indexContent = await fs.readFile(path.join(snapshotDir, "index.json"), "utf-8");
|
|
63
|
-
const index = JSON.parse(indexContent);
|
|
64
|
-
expect(index.metadata.totalPages).toBe(1);
|
|
65
|
-
expect(Object.keys(index.urls)).toHaveLength(1);
|
|
66
|
-
});
|
|
67
|
-
it("should handle empty dataset", async () => {
|
|
68
|
-
const emptyDataset = await Dataset.open("empty-dataset");
|
|
69
|
-
const metadata = {
|
|
70
|
-
baseUrl: "https://example.com",
|
|
71
|
-
timestamp: new Date().toISOString(),
|
|
72
|
-
crawlSettings: {
|
|
73
|
-
baseUrl: "https://example.com",
|
|
74
|
-
crawlerType: "cheerio",
|
|
75
|
-
maxRequestsPerCrawl: 10,
|
|
76
|
-
},
|
|
77
|
-
};
|
|
78
|
-
const pageCount = await snapshotRepository.saveSnapshot("empty-snapshot", {
|
|
79
|
-
dataset: emptyDataset,
|
|
80
|
-
metadata,
|
|
81
|
-
});
|
|
82
|
-
expect(pageCount).toBe(0);
|
|
83
|
-
await emptyDataset.drop();
|
|
84
|
-
});
|
|
85
|
-
});
|
|
86
|
-
describe("loadSnapshot", () => {
|
|
87
|
-
it("should load a saved snapshot", async () => {
|
|
88
|
-
// First save a snapshot
|
|
89
|
-
const metadata = {
|
|
90
|
-
baseUrl: "https://example.com",
|
|
91
|
-
timestamp: new Date().toISOString(),
|
|
92
|
-
crawlSettings: {
|
|
93
|
-
baseUrl: "https://example.com",
|
|
94
|
-
crawlerType: "cheerio",
|
|
95
|
-
maxRequestsPerCrawl: 10,
|
|
96
|
-
},
|
|
97
|
-
};
|
|
98
|
-
await snapshotRepository.saveSnapshot("test-snapshot", {
|
|
99
|
-
dataset: testDataset,
|
|
100
|
-
metadata,
|
|
101
|
-
});
|
|
102
|
-
// Then load it
|
|
103
|
-
const loadedSnapshot = await snapshotRepository.loadSnapshot("test-snapshot");
|
|
104
|
-
expect(loadedSnapshot).toBeInstanceOf(LoadedSnapshot);
|
|
105
|
-
expect(loadedSnapshot.metadata).toEqual(metadata);
|
|
106
|
-
expect(loadedSnapshot.index.metadata.totalPages).toBe(1);
|
|
107
|
-
// Test loading a specific page
|
|
108
|
-
const page = await loadedSnapshot.getPage("/");
|
|
109
|
-
expect(page).not.toBeNull();
|
|
110
|
-
expect(page?.url).toBe("/");
|
|
111
|
-
expect(page?.statusCode).toBe(200);
|
|
112
|
-
});
|
|
113
|
-
it("should return null for non-existent page", async () => {
|
|
114
|
-
// First save a snapshot
|
|
115
|
-
const metadata = {
|
|
116
|
-
baseUrl: "https://example.com",
|
|
117
|
-
timestamp: new Date().toISOString(),
|
|
118
|
-
crawlSettings: {
|
|
119
|
-
baseUrl: "https://example.com",
|
|
120
|
-
crawlerType: "cheerio",
|
|
121
|
-
maxRequestsPerCrawl: 10,
|
|
122
|
-
},
|
|
123
|
-
};
|
|
124
|
-
await snapshotRepository.saveSnapshot("test-snapshot", {
|
|
125
|
-
dataset: testDataset,
|
|
126
|
-
metadata,
|
|
127
|
-
});
|
|
128
|
-
// Then load it
|
|
129
|
-
const loadedSnapshot = await snapshotRepository.loadSnapshot("test-snapshot");
|
|
130
|
-
// Try to get a non-existent page
|
|
131
|
-
const page = await loadedSnapshot.getPage("/nonexistent");
|
|
132
|
-
expect(page).toBeNull();
|
|
133
|
-
});
|
|
134
|
-
});
|
|
135
|
-
describe("generateUrlList", () => {
|
|
136
|
-
it("should generate a URL list file", async () => {
|
|
137
|
-
// First save a snapshot
|
|
138
|
-
const metadata = {
|
|
139
|
-
baseUrl: "https://example.com",
|
|
140
|
-
timestamp: new Date().toISOString(),
|
|
141
|
-
crawlSettings: {
|
|
142
|
-
baseUrl: "https://example.com",
|
|
143
|
-
crawlerType: "cheerio",
|
|
144
|
-
maxRequestsPerCrawl: 10,
|
|
145
|
-
},
|
|
146
|
-
};
|
|
147
|
-
await snapshotRepository.saveSnapshot("test-snapshot", {
|
|
148
|
-
dataset: testDataset,
|
|
149
|
-
metadata,
|
|
150
|
-
});
|
|
151
|
-
// Generate URL list
|
|
152
|
-
const outputPath = await snapshotRepository.generateUrlList("test-snapshot");
|
|
153
|
-
// Verify file exists and contains correct content
|
|
154
|
-
const content = await fs.readFile(outputPath, "utf-8");
|
|
155
|
-
expect(content.trim()).toBe("/");
|
|
156
|
-
});
|
|
157
|
-
it("should filter URLs based on status code", async () => {
|
|
158
|
-
// Add a page with error status
|
|
159
|
-
await testDataset.pushData({
|
|
160
|
-
type: "page",
|
|
161
|
-
url: "https://example.com/error",
|
|
162
|
-
finalUrl: "https://example.com/error",
|
|
163
|
-
statusCode: 404,
|
|
164
|
-
title: "Error Page",
|
|
165
|
-
html: "<html><body>Error</body></html>",
|
|
166
|
-
headers: { "content-type": "text/html" },
|
|
167
|
-
});
|
|
168
|
-
const metadata = {
|
|
169
|
-
baseUrl: "https://example.com",
|
|
170
|
-
timestamp: new Date().toISOString(),
|
|
171
|
-
crawlSettings: {
|
|
172
|
-
baseUrl: "https://example.com",
|
|
173
|
-
crawlerType: "cheerio",
|
|
174
|
-
maxRequestsPerCrawl: 10,
|
|
175
|
-
},
|
|
176
|
-
};
|
|
177
|
-
await snapshotRepository.saveSnapshot("test-snapshot", {
|
|
178
|
-
dataset: testDataset,
|
|
179
|
-
metadata,
|
|
180
|
-
});
|
|
181
|
-
// Generate URL list with filter
|
|
182
|
-
const outputPath = await snapshotRepository.generateUrlList("test-snapshot", undefined, (url, statusCode) => statusCode === 200);
|
|
183
|
-
const content = await fs.readFile(outputPath, "utf-8");
|
|
184
|
-
const urls = content.trim().split("\n");
|
|
185
|
-
expect(urls).toHaveLength(1);
|
|
186
|
-
expect(urls[0]).toBe("/");
|
|
187
|
-
});
|
|
188
|
-
});
|
|
189
|
-
describe("listSnapshots", () => {
|
|
190
|
-
it("should list all snapshots", async () => {
|
|
191
|
-
// Create multiple snapshots
|
|
192
|
-
const metadata = {
|
|
193
|
-
baseUrl: "https://example.com",
|
|
194
|
-
timestamp: new Date().toISOString(),
|
|
195
|
-
crawlSettings: {
|
|
196
|
-
baseUrl: "https://example.com",
|
|
197
|
-
crawlerType: "cheerio",
|
|
198
|
-
maxRequestsPerCrawl: 10,
|
|
199
|
-
},
|
|
200
|
-
};
|
|
201
|
-
await snapshotRepository.saveSnapshot("snapshot1", {
|
|
202
|
-
dataset: testDataset,
|
|
203
|
-
metadata,
|
|
204
|
-
});
|
|
205
|
-
await snapshotRepository.saveSnapshot("snapshot2", {
|
|
206
|
-
dataset: testDataset,
|
|
207
|
-
metadata,
|
|
208
|
-
});
|
|
209
|
-
const snapshots = await snapshotRepository.listSnapshots();
|
|
210
|
-
// Check that we got the expected number of snapshots
|
|
211
|
-
expect(snapshots).toHaveLength(2);
|
|
212
|
-
// Check that both snapshots are present with correct data
|
|
213
|
-
const snapshot1 = snapshots.find((s) => s.name === "snapshot1");
|
|
214
|
-
const snapshot2 = snapshots.find((s) => s.name === "snapshot2");
|
|
215
|
-
expect(snapshot1).toBeDefined();
|
|
216
|
-
expect(snapshot2).toBeDefined();
|
|
217
|
-
// Check snapshot1 details
|
|
218
|
-
expect(snapshot1).toEqual({
|
|
219
|
-
name: "snapshot1",
|
|
220
|
-
date: metadata.timestamp,
|
|
221
|
-
pageCount: 1,
|
|
222
|
-
errorCount: 0,
|
|
223
|
-
});
|
|
224
|
-
// Check snapshot2 details
|
|
225
|
-
expect(snapshot2).toEqual({
|
|
226
|
-
name: "snapshot2",
|
|
227
|
-
date: metadata.timestamp,
|
|
228
|
-
pageCount: 1,
|
|
229
|
-
errorCount: 0,
|
|
230
|
-
});
|
|
231
|
-
});
|
|
232
|
-
it("should return empty array when no snapshots exist", async () => {
|
|
233
|
-
const snapshots = await snapshotRepository.listSnapshots();
|
|
234
|
-
expect(snapshots).toEqual([]);
|
|
235
|
-
});
|
|
236
|
-
});
|
|
237
|
-
});
|
|
238
|
-
//# sourceMappingURL=index.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../../../src/__tests__/snapshot/index.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AACxD,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAGzE,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IAClC,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,gBAAgB,CAAC,CAAC;IACtE,IAAI,kBAAsC,CAAC;IAC3C,IAAI,WAAoB,CAAC;IAEzB,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,4BAA4B;QAC5B,MAAM,oBAAoB,EAAE,CAAC;QAE7B,0CAA0C;QAC1C,MAAM,EAAE,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,kBAAkB,GAAG,MAAM,kBAAkB,CAAC,mBAAmB,CAC/D,kBAAkB,CACnB,CAAC;QAEF,wBAAwB;QACxB,WAAW,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACjD,sBAAsB;QACtB,MAAM,WAAW,CAAC,QAAQ,CAAC;YACzB,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,qBAAqB;YAC1B,QAAQ,EAAE,qBAAqB;YAC/B,UAAU,EAAE,GAAG;YACf,KAAK,EAAE,cAAc;YACrB,IAAI,EAAE,gCAAgC;YACtC,OAAO,EAAE,EAAE,cAAc,EAAE,WAAW,EAAE;SACzC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,oCAAoC;QACpC,MAAM,EAAE,CAAC,EAAE,CAAC,kBAAkB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAClE,mBAAmB;QACnB,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,QAAQ,GAAG;gBACf,OAAO,EAAE,qBAAqB;gBAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,aAAa,EAAE;oBACb,OAAO,EAAE,qBAAqB;oBAC9B,WAAW,EAAE,SAAwB;oBACrC,mBAAmB,EAAE,EAAE;iBACxB;aACF,CAAC;YAEF,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAAC,eAAe,EAAE;gBACvE,OAAO,EAAE,WAAW;gBACpB,QAAQ;aACT,CAAC,CAAC;YAEH,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAE1B,4BAA4B;YAC5B,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC;YACnE,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAC5C,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;YACzC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACtC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YAEjC,kBAAkB;YAClB,MAAM,eAAe,GAAG,MAAM,EAAE,CAAC,QAAQ,CACvC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,eAAe,CAAC,EACvC,OAAO,CACR,CAAC;YACF,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAClD,MAAM,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAExC,eAAe;YACf,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC,QAAQ,CACpC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC,EACpC,OAAO,CACR,CAAC;YACF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACvC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC3C,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACzD,MAAM,QAAQ,GAAG;gBACf,OAAO,EAAE,qBAAqB;gBAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,aAAa,EAAE;oBACb,OAAO,EAAE,qBAAqB;oBAC9B,WAAW,EAAE,SAAwB;oBACrC,mBAAmB,EAAE,EAAE;iBACxB;aACF,CAAC;YAEF,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC,YAAY,CACrD,gBAAgB,EAChB;gBACE,OAAO,EAAE,YAAY;gBACrB,QAAQ;aACT,CACF,CAAC;YAEF,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,YAAY,CAAC,IAAI,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,wBAAwB;YACxB,MAAM,QAAQ,GAAG;gBACf,OAAO,EAAE,qBAAqB;gBAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,aAAa,EAAE;oBACb,OAAO,EAAE,qBAAqB;oBAC9B,WAAW,EAAE,SAAwB;oBACrC,mBAAmB,EAAE,EAAE;iBACxB;aACF,CAAC;YAEF,MAAM,kBAAkB,CAAC,YAAY,CAAC,eAAe,EAAE;gBACrD,OAAO,EAAE,WAAW;gBACpB,QAAQ;aACT,CAAC,CAAC;YAEH,eAAe;YACf,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAC1D,eAAe,CAChB,CAAC;YAEF,MAAM,CAAC,cAAc,CAAC,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;YACtD,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAClD,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAEzD,+BAA+B;YAC/B,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC/C,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,wBAAwB;YACxB,MAAM,QAAQ,GAAG;gBACf,OAAO,EAAE,qBAAqB;gBAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,aAAa,EAAE;oBACb,OAAO,EAAE,qBAAqB;oBAC9B,WAAW,EAAE,SAAwB;oBACrC,mBAAmB,EAAE,EAAE;iBACxB;aACF,CAAC;YAEF,MAAM,kBAAkB,CAAC,YAAY,CAAC,eAAe,EAAE;gBACrD,OAAO,EAAE,WAAW;gBACpB,QAAQ;aACT,CAAC,CAAC;YAEH,eAAe;YACf,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAC1D,eAAe,CAChB,CAAC;YAEF,iCAAiC;YACjC,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAC1D,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,wBAAwB;YACxB,MAAM,QAAQ,GAAG;gBACf,OAAO,EAAE,qBAAqB;gBAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,aAAa,EAAE;oBACb,OAAO,EAAE,qBAAqB;oBAC9B,WAAW,EAAE,SAAwB;oBACrC,mBAAmB,EAAE,EAAE;iBACxB;aACF,CAAC;YAEF,MAAM,kBAAkB,CAAC,YAAY,CAAC,eAAe,EAAE;gBACrD,OAAO,EAAE,WAAW;gBACpB,QAAQ;aACT,CAAC,CAAC;YAEH,oBAAoB;YACpB,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,eAAe,CACzD,eAAe,CAChB,CAAC;YAEF,kDAAkD;YAClD,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACvD,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,+BAA+B;YAC/B,MAAM,WAAW,CAAC,QAAQ,CAAC;gBACzB,IAAI,EAAE,MAAM;gBACZ,GAAG,EAAE,2BAA2B;gBAChC,QAAQ,EAAE,2BAA2B;gBACrC,UAAU,EAAE,GAAG;gBACf,KAAK,EAAE,YAAY;gBACnB,IAAI,EAAE,iCAAiC;gBACvC,OAAO,EAAE,EAAE,cAAc,EAAE,WAAW,EAAE;aACzC,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG;gBACf,OAAO,EAAE,qBAAqB;gBAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,aAAa,EAAE;oBACb,OAAO,EAAE,qBAAqB;oBAC9B,WAAW,EAAE,SAAwB;oBACrC,mBAAmB,EAAE,EAAE;iBACxB;aACF,CAAC;YAEF,MAAM,kBAAkB,CAAC,YAAY,CAAC,eAAe,EAAE;gBACrD,OAAO,EAAE,WAAW;gBACpB,QAAQ;aACT,CAAC,CAAC;YAEH,gCAAgC;YAChC,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,eAAe,CACzD,eAAe,EACf,SAAS,EACT,CAAC,GAAG,EAAE,UAAU,EAAE,EAAE,CAAC,UAAU,KAAK,GAAG,CACxC,CAAC;YAEF,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACvD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;YACzC,4BAA4B;YAC5B,MAAM,QAAQ,GAAG;gBACf,OAAO,EAAE,qBAAqB;gBAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,aAAa,EAAE;oBACb,OAAO,EAAE,qBAAqB;oBAC9B,WAAW,EAAE,SAAwB;oBACrC,mBAAmB,EAAE,EAAE;iBACxB;aACF,CAAC;YAEF,MAAM,kBAAkB,CAAC,YAAY,CAAC,WAAW,EAAE;gBACjD,OAAO,EAAE,WAAW;gBACpB,QAAQ;aACT,CAAC,CAAC;YAEH,MAAM,kBAAkB,CAAC,YAAY,CAAC,WAAW,EAAE;gBACjD,OAAO,EAAE,WAAW;gBACpB,QAAQ;aACT,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC,aAAa,EAAE,CAAC;YAE3D,qDAAqD;YACrD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAElC,0DAA0D;YAC1D,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;YAChE,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;YAEhE,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;YAChC,MAAM,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;YAEhC,0BAA0B;YAC1B,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC;gBACxB,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,QAAQ,CAAC,SAAS;gBACxB,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC;aACd,CAAC,CAAC;YAEH,0BAA0B;YAC1B,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC;gBACxB,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,QAAQ,CAAC,SAAS;gBACxB,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC;aACd,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC,aAAa,EAAE,CAAC;YAC3D,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1,102 +0,0 @@
|
|
|
1
|
-
import http from "http";
|
|
2
|
-
import { promisify } from "util";
|
|
3
|
-
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
4
|
-
// Mock fs/promises before importing the module
|
|
5
|
-
vi.mock("fs/promises", () => ({
|
|
6
|
-
default: {
|
|
7
|
-
readFile: vi.fn(async (path) => {
|
|
8
|
-
if (typeof path === "string") {
|
|
9
|
-
if (path.includes("package.json")) {
|
|
10
|
-
return JSON.stringify({
|
|
11
|
-
workspaces: ["packages/*"],
|
|
12
|
-
name: "breakcheck-monorepo",
|
|
13
|
-
});
|
|
14
|
-
}
|
|
15
|
-
if (path.includes("index.json")) {
|
|
16
|
-
return JSON.stringify({
|
|
17
|
-
urls: {
|
|
18
|
-
"/page1": { filename: "page1.json.gz", hasDifferences: true },
|
|
19
|
-
"/page2": { filename: "page2.json.gz", hasDifferences: false },
|
|
20
|
-
},
|
|
21
|
-
});
|
|
22
|
-
}
|
|
23
|
-
if (path.includes("diffs/page1.json.gz")) {
|
|
24
|
-
return Buffer.from("compressed data");
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
throw new Error(`fs.readFile mock not implemented for ${path}`);
|
|
28
|
-
}),
|
|
29
|
-
},
|
|
30
|
-
}));
|
|
31
|
-
vi.mock("zlib", () => ({
|
|
32
|
-
gunzip: vi.fn((_buffer, callback) => {
|
|
33
|
-
const decompressed = Buffer.from(JSON.stringify({
|
|
34
|
-
url: "/page1",
|
|
35
|
-
differences: [],
|
|
36
|
-
hasDifferences: true,
|
|
37
|
-
}));
|
|
38
|
-
callback(null, decompressed);
|
|
39
|
-
}),
|
|
40
|
-
}));
|
|
41
|
-
// Import after mocks
|
|
42
|
-
import { startViewServer } from "../../core/view";
|
|
43
|
-
describe("View Server", () => {
|
|
44
|
-
const mockComparisonName = "test-comparison";
|
|
45
|
-
const mockPort = 8080;
|
|
46
|
-
let server;
|
|
47
|
-
const closeServer = promisify((server, cb) => server.close(cb));
|
|
48
|
-
beforeEach(async () => {
|
|
49
|
-
vi.clearAllMocks();
|
|
50
|
-
try {
|
|
51
|
-
server = await startViewServer(mockComparisonName, mockPort);
|
|
52
|
-
}
|
|
53
|
-
catch (error) {
|
|
54
|
-
console.error("Failed to start server:", error);
|
|
55
|
-
throw error;
|
|
56
|
-
}
|
|
57
|
-
});
|
|
58
|
-
afterEach(async () => {
|
|
59
|
-
if (server) {
|
|
60
|
-
try {
|
|
61
|
-
await closeServer(server);
|
|
62
|
-
}
|
|
63
|
-
catch (error) {
|
|
64
|
-
// Ignore errors when server is already closed
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
});
|
|
68
|
-
const makeRequest = (path) => {
|
|
69
|
-
return new Promise((resolve, reject) => {
|
|
70
|
-
const options = {
|
|
71
|
-
host: "localhost",
|
|
72
|
-
path,
|
|
73
|
-
port: mockPort,
|
|
74
|
-
agent: new http.Agent({ keepAlive: false }),
|
|
75
|
-
};
|
|
76
|
-
http
|
|
77
|
-
.get(options, (res) => {
|
|
78
|
-
let data = "";
|
|
79
|
-
res.on("data", (chunk) => {
|
|
80
|
-
data += chunk;
|
|
81
|
-
});
|
|
82
|
-
res.on("end", () => {
|
|
83
|
-
resolve({ statusCode: res.statusCode || 500, data });
|
|
84
|
-
});
|
|
85
|
-
})
|
|
86
|
-
.on("error", reject);
|
|
87
|
-
});
|
|
88
|
-
};
|
|
89
|
-
it("should start the server and respond to the index route", async () => {
|
|
90
|
-
const response = await makeRequest("/");
|
|
91
|
-
expect(response.statusCode).toBe(200);
|
|
92
|
-
});
|
|
93
|
-
it("should handle diff route correctly", async () => {
|
|
94
|
-
const response = await makeRequest("/diff?page=/page1");
|
|
95
|
-
expect(response.statusCode).toBe(200);
|
|
96
|
-
});
|
|
97
|
-
it("should handle missing page parameter in diff route", async () => {
|
|
98
|
-
const response = await makeRequest("/diff");
|
|
99
|
-
expect(response.statusCode).toBe(400);
|
|
100
|
-
});
|
|
101
|
-
});
|
|
102
|
-
//# sourceMappingURL=index.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../../../src/__tests__/view/index.test.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAEzE,+CAA+C;AAC/C,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5B,OAAO,EAAE;QACP,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,IAA2B,EAAE,EAAE;YACpD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,IAAI,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;oBAClC,OAAO,IAAI,CAAC,SAAS,CAAC;wBACpB,UAAU,EAAE,CAAC,YAAY,CAAC;wBAC1B,IAAI,EAAE,qBAAqB;qBAC5B,CAAC,CAAC;gBACL,CAAC;gBACD,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBAChC,OAAO,IAAI,CAAC,SAAS,CAAC;wBACpB,IAAI,EAAE;4BACJ,QAAQ,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,cAAc,EAAE,IAAI,EAAE;4BAC7D,QAAQ,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,cAAc,EAAE,KAAK,EAAE;yBAC/D;qBACF,CAAC,CAAC;gBACL,CAAC;gBACD,IAAI,IAAI,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC;oBACzC,OAAO,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBACxC,CAAC;YACH,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC,CAAC;KACH;CACF,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IACrB,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE;QAClC,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAC9B,IAAI,CAAC,SAAS,CAAC;YACb,GAAG,EAAE,QAAQ;YACb,WAAW,EAAE,EAAE;YACf,cAAc,EAAE,IAAI;SACrB,CAAC,CACH,CAAC;QACF,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IAC/B,CAAC,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,qBAAqB;AACrB,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAElD,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,MAAM,kBAAkB,GAAG,iBAAiB,CAAC;IAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC;IACtB,IAAI,MAA+B,CAAC;IACpC,MAAM,WAAW,GAAG,SAAS,CAC3B,CAAC,MAAmB,EAAE,EAAyB,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CACrE,CAAC;IAEF,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,eAAe,CAAC,kBAAkB,EAAE,QAAQ,CAAC,CAAC;QAC/D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAC;YAChD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,8CAA8C;YAChD,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,CAClB,IAAY,EACmC,EAAE;QACjD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,OAAO,GAAG;gBACd,IAAI,EAAE,WAAW;gBACjB,IAAI;gBACJ,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;aAC5C,CAAC;YACF,IAAI;iBACD,GAAG,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACpB,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;oBACvB,IAAI,IAAI,KAAK,CAAC;gBAChB,CAAC,CAAC,CAAC;gBACH,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACjB,OAAO,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,IAAI,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;gBACvD,CAAC,CAAC,CAAC;YACL,CAAC,CAAC;iBACD,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;QACtE,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,mBAAmB,CAAC,CAAC;QACxD,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;QAC5C,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|