@forwardimpact/libutil 0.1.72 → 0.1.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/fit-download-bundle.js +1 -1
- package/bin/fit-tiktoken.js +1 -1
- package/package.json +12 -2
- package/{finder.js → src/finder.js} +4 -1
- package/test/downloader.test.js +0 -223
- package/test/extractor.test.js +0 -338
- package/test/finder.test.js +0 -358
- package/test/fixtures/sample.tar.gz +0 -0
- package/test/fixtures/sample.zip +0 -0
- package/test/http.test.js +0 -93
- package/test/libutil.test.js +0 -35
- package/test/logger.test.js +0 -219
- package/test/processor.test.js +0 -140
- package/test/retry.test.js +0 -194
- package/test/tokenizer.test.js +0 -123
- package/test/wait.test.js +0 -109
- /package/{downloader.js → src/downloader.js} +0 -0
- /package/{extractor.js → src/extractor.js} +0 -0
- /package/{http.js → src/http.js} +0 -0
- /package/{index.js → src/index.js} +0 -0
- /package/{processor.js → src/processor.js} +0 -0
- /package/{retry.js → src/retry.js} +0 -0
- /package/{tokenizer.js → src/tokenizer.js} +0 -0
- /package/{wait.js → src/wait.js} +0 -0
package/test/logger.test.js
DELETED
|
@@ -1,219 +0,0 @@
|
|
|
1
|
-
import { test, describe, beforeEach, afterEach } from "node:test";
|
|
2
|
-
import assert from "node:assert";
|
|
3
|
-
|
|
4
|
-
// Module under test
|
|
5
|
-
import { Logger, createLogger } from "@forwardimpact/libtelemetry";
|
|
6
|
-
|
|
7
|
-
describe("Logger", () => {
|
|
8
|
-
let originalDebug;
|
|
9
|
-
let consoleOutput;
|
|
10
|
-
let originalConsoleError;
|
|
11
|
-
|
|
12
|
-
beforeEach(() => {
|
|
13
|
-
originalDebug = process.env.DEBUG;
|
|
14
|
-
consoleOutput = [];
|
|
15
|
-
originalConsoleError = console.error;
|
|
16
|
-
console.error = (message) => consoleOutput.push(message);
|
|
17
|
-
});
|
|
18
|
-
|
|
19
|
-
afterEach(() => {
|
|
20
|
-
process.env.DEBUG = originalDebug;
|
|
21
|
-
console.error = originalConsoleError;
|
|
22
|
-
});
|
|
23
|
-
|
|
24
|
-
test("creates Logger with domain", () => {
|
|
25
|
-
const logger = new Logger("test");
|
|
26
|
-
|
|
27
|
-
assert.ok(logger instanceof Logger);
|
|
28
|
-
assert.strictEqual(logger.domain, "test");
|
|
29
|
-
});
|
|
30
|
-
|
|
31
|
-
test("validates constructor parameters", () => {
|
|
32
|
-
assert.throws(() => new Logger(), {
|
|
33
|
-
message: /domain must be a non-empty string/,
|
|
34
|
-
});
|
|
35
|
-
assert.throws(() => new Logger(""), {
|
|
36
|
-
message: /domain must be a non-empty string/,
|
|
37
|
-
});
|
|
38
|
-
assert.throws(() => new Logger(null), {
|
|
39
|
-
message: /domain must be a non-empty string/,
|
|
40
|
-
});
|
|
41
|
-
});
|
|
42
|
-
|
|
43
|
-
test("enables logging when DEBUG=*", () => {
|
|
44
|
-
process.env.DEBUG = "*";
|
|
45
|
-
const logger = new Logger("test");
|
|
46
|
-
|
|
47
|
-
assert.strictEqual(logger.enabled, true);
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
test("disables logging when DEBUG is empty", () => {
|
|
51
|
-
process.env.DEBUG = "";
|
|
52
|
-
const logger = new Logger("test");
|
|
53
|
-
|
|
54
|
-
assert.strictEqual(logger.enabled, false);
|
|
55
|
-
});
|
|
56
|
-
|
|
57
|
-
test("enables logging for exact domain match", () => {
|
|
58
|
-
process.env.DEBUG = "test,other";
|
|
59
|
-
const logger = new Logger("test");
|
|
60
|
-
|
|
61
|
-
assert.strictEqual(logger.enabled, true);
|
|
62
|
-
});
|
|
63
|
-
|
|
64
|
-
test("enables logging for wildcard pattern match", () => {
|
|
65
|
-
process.env.DEBUG = "test*";
|
|
66
|
-
const logger = new Logger("test:service");
|
|
67
|
-
|
|
68
|
-
assert.strictEqual(logger.enabled, true);
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
test("disables logging for non-matching domain", () => {
|
|
72
|
-
process.env.DEBUG = "other";
|
|
73
|
-
const logger = new Logger("test");
|
|
74
|
-
|
|
75
|
-
assert.strictEqual(logger.enabled, false);
|
|
76
|
-
});
|
|
77
|
-
|
|
78
|
-
test("logs debug message when enabled", () => {
|
|
79
|
-
process.env.DEBUG = "test";
|
|
80
|
-
const logger = new Logger("test");
|
|
81
|
-
|
|
82
|
-
logger.debug("TestApp", "Test message");
|
|
83
|
-
|
|
84
|
-
assert.strictEqual(consoleOutput.length, 1);
|
|
85
|
-
assert.ok(consoleOutput[0].includes("DEBUG"));
|
|
86
|
-
assert.ok(consoleOutput[0].includes("test"));
|
|
87
|
-
assert.ok(consoleOutput[0].includes("TestApp"));
|
|
88
|
-
assert.ok(consoleOutput[0].includes("Test message"));
|
|
89
|
-
});
|
|
90
|
-
|
|
91
|
-
test("does not log when disabled", () => {
|
|
92
|
-
process.env.DEBUG = "other";
|
|
93
|
-
const logger = new Logger("test");
|
|
94
|
-
|
|
95
|
-
logger.debug("TestApp", "Test message");
|
|
96
|
-
|
|
97
|
-
assert.strictEqual(consoleOutput.length, 0);
|
|
98
|
-
});
|
|
99
|
-
|
|
100
|
-
test("handles empty data object", () => {
|
|
101
|
-
process.env.DEBUG = "test";
|
|
102
|
-
const logger = new Logger("test");
|
|
103
|
-
|
|
104
|
-
logger.debug("TestApp", "Test message", {});
|
|
105
|
-
|
|
106
|
-
assert.strictEqual(consoleOutput.length, 1);
|
|
107
|
-
assert.ok(consoleOutput[0].includes("DEBUG"));
|
|
108
|
-
assert.ok(consoleOutput[0].includes("Test message"));
|
|
109
|
-
});
|
|
110
|
-
|
|
111
|
-
test("includes timestamp in log output", () => {
|
|
112
|
-
process.env.DEBUG = "test";
|
|
113
|
-
const logger = new Logger("test");
|
|
114
|
-
|
|
115
|
-
logger.debug("TestApp", "Test message");
|
|
116
|
-
|
|
117
|
-
assert.strictEqual(consoleOutput.length, 1);
|
|
118
|
-
assert.ok(consoleOutput[0].match(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/));
|
|
119
|
-
});
|
|
120
|
-
|
|
121
|
-
test("merges trace context with provided attributes", () => {
|
|
122
|
-
process.env.DEBUG = "test";
|
|
123
|
-
const logger = new Logger("test");
|
|
124
|
-
|
|
125
|
-
const error = new Error("Test error");
|
|
126
|
-
Object.defineProperty(error, "trace_id", {
|
|
127
|
-
value: "trace123",
|
|
128
|
-
enumerable: false,
|
|
129
|
-
writable: false,
|
|
130
|
-
});
|
|
131
|
-
|
|
132
|
-
logger.error("TestMethod", error, { retry: "1/3", status: "500" });
|
|
133
|
-
|
|
134
|
-
assert.strictEqual(consoleOutput.length, 1);
|
|
135
|
-
assert.ok(consoleOutput[0].includes('trace_id="trace123"'));
|
|
136
|
-
assert.ok(consoleOutput[0].includes('retry="1/3"'));
|
|
137
|
-
assert.ok(consoleOutput[0].includes('status="500"'));
|
|
138
|
-
});
|
|
139
|
-
|
|
140
|
-
test("exception logs message when disabled", () => {
|
|
141
|
-
process.env.DEBUG = "other";
|
|
142
|
-
const logger = new Logger("test");
|
|
143
|
-
|
|
144
|
-
const error = new Error("Test error");
|
|
145
|
-
|
|
146
|
-
logger.exception("TestMethod", error);
|
|
147
|
-
|
|
148
|
-
assert.strictEqual(consoleOutput.length, 1);
|
|
149
|
-
assert.ok(consoleOutput[0].includes("ERROR"));
|
|
150
|
-
assert.ok(consoleOutput[0].includes("Test error"));
|
|
151
|
-
assert.ok(
|
|
152
|
-
!consoleOutput[0].includes("at "),
|
|
153
|
-
"Should not include stack trace when disabled",
|
|
154
|
-
);
|
|
155
|
-
});
|
|
156
|
-
|
|
157
|
-
test("exception logs message with stack trace when enabled", () => {
|
|
158
|
-
process.env.DEBUG = "test";
|
|
159
|
-
const logger = new Logger("test");
|
|
160
|
-
|
|
161
|
-
const error = new Error("Test error");
|
|
162
|
-
|
|
163
|
-
logger.exception("TestMethod", error);
|
|
164
|
-
|
|
165
|
-
assert.strictEqual(consoleOutput.length, 1);
|
|
166
|
-
assert.ok(consoleOutput[0].includes("ERROR"));
|
|
167
|
-
assert.ok(consoleOutput[0].includes("Test error"));
|
|
168
|
-
assert.ok(
|
|
169
|
-
consoleOutput[0].includes("at "),
|
|
170
|
-
"Should include stack trace when enabled",
|
|
171
|
-
);
|
|
172
|
-
});
|
|
173
|
-
|
|
174
|
-
test("exception extracts trace context from error", () => {
|
|
175
|
-
process.env.DEBUG = "test";
|
|
176
|
-
const logger = new Logger("test");
|
|
177
|
-
|
|
178
|
-
const error = new Error("Test error");
|
|
179
|
-
error.trace_id = "trace456";
|
|
180
|
-
error.span_id = "span789";
|
|
181
|
-
error.service_name = "my-service";
|
|
182
|
-
|
|
183
|
-
logger.exception("TestMethod", error);
|
|
184
|
-
|
|
185
|
-
assert.strictEqual(consoleOutput.length, 1);
|
|
186
|
-
assert.ok(consoleOutput[0].includes('trace_id="trace456"'));
|
|
187
|
-
assert.ok(consoleOutput[0].includes('span_id="span789"'));
|
|
188
|
-
assert.ok(consoleOutput[0].includes('service_name="my-service"'));
|
|
189
|
-
});
|
|
190
|
-
|
|
191
|
-
test("exception merges trace context with provided attributes", () => {
|
|
192
|
-
process.env.DEBUG = "test";
|
|
193
|
-
const logger = new Logger("test");
|
|
194
|
-
|
|
195
|
-
const error = new Error("Test error");
|
|
196
|
-
error.trace_id = "trace123";
|
|
197
|
-
|
|
198
|
-
logger.exception("TestMethod", error, { retry: "2/3" });
|
|
199
|
-
|
|
200
|
-
assert.strictEqual(consoleOutput.length, 1);
|
|
201
|
-
assert.ok(consoleOutput[0].includes('trace_id="trace123"'));
|
|
202
|
-
assert.ok(consoleOutput[0].includes('retry="2/3"'));
|
|
203
|
-
});
|
|
204
|
-
});
|
|
205
|
-
|
|
206
|
-
describe("createLogger", () => {
|
|
207
|
-
test("creates Logger instance", () => {
|
|
208
|
-
const logger = createLogger("test");
|
|
209
|
-
|
|
210
|
-
assert.ok(logger instanceof Logger);
|
|
211
|
-
assert.strictEqual(logger.domain, "test");
|
|
212
|
-
});
|
|
213
|
-
|
|
214
|
-
test("passes through domain validation", () => {
|
|
215
|
-
assert.throws(() => createLogger(""), {
|
|
216
|
-
message: /domain must be a non-empty string/,
|
|
217
|
-
});
|
|
218
|
-
});
|
|
219
|
-
});
|
package/test/processor.test.js
DELETED
|
@@ -1,140 +0,0 @@
|
|
|
1
|
-
import { test, describe, beforeEach } from "node:test";
|
|
2
|
-
import assert from "node:assert";
|
|
3
|
-
|
|
4
|
-
import { createSilentLogger } from "@forwardimpact/libharness";
|
|
5
|
-
|
|
6
|
-
// Module under test
|
|
7
|
-
import { ProcessorBase } from "../processor.js";
|
|
8
|
-
|
|
9
|
-
describe("ProcessorBase", () => {
|
|
10
|
-
let mockLogger;
|
|
11
|
-
|
|
12
|
-
beforeEach(() => {
|
|
13
|
-
mockLogger = createSilentLogger();
|
|
14
|
-
});
|
|
15
|
-
|
|
16
|
-
describe("constructor", () => {
|
|
17
|
-
test("creates ProcessorBase with logger and batch size", () => {
|
|
18
|
-
const processor = new ProcessorBase(mockLogger, 5);
|
|
19
|
-
|
|
20
|
-
assert.ok(processor instanceof ProcessorBase);
|
|
21
|
-
});
|
|
22
|
-
|
|
23
|
-
test("validates logger parameter", () => {
|
|
24
|
-
assert.throws(() => new ProcessorBase(), {
|
|
25
|
-
message: /logger is required/,
|
|
26
|
-
});
|
|
27
|
-
assert.throws(() => new ProcessorBase(null), {
|
|
28
|
-
message: /logger is required/,
|
|
29
|
-
});
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
test("validates batch size parameter", () => {
|
|
33
|
-
assert.throws(() => new ProcessorBase(mockLogger, 0), {
|
|
34
|
-
message: /batchSize must be a positive number/,
|
|
35
|
-
});
|
|
36
|
-
assert.throws(() => new ProcessorBase(mockLogger, -1), {
|
|
37
|
-
message: /batchSize must be a positive number/,
|
|
38
|
-
});
|
|
39
|
-
assert.throws(() => new ProcessorBase(mockLogger, "invalid"), {
|
|
40
|
-
message: /batchSize must be a positive number/,
|
|
41
|
-
});
|
|
42
|
-
});
|
|
43
|
-
|
|
44
|
-
test("uses default batch size when not provided", () => {
|
|
45
|
-
const processor = new ProcessorBase(mockLogger);
|
|
46
|
-
// Test passes if no error is thrown
|
|
47
|
-
assert.ok(processor instanceof ProcessorBase);
|
|
48
|
-
});
|
|
49
|
-
});
|
|
50
|
-
|
|
51
|
-
describe("process", () => {
|
|
52
|
-
test("validates items parameter", async () => {
|
|
53
|
-
const processor = new ProcessorBase(mockLogger, 2);
|
|
54
|
-
|
|
55
|
-
await assert.rejects(() => processor.process("not-array"), {
|
|
56
|
-
message: /items must be an array/,
|
|
57
|
-
});
|
|
58
|
-
});
|
|
59
|
-
|
|
60
|
-
test("handles empty array", async () => {
|
|
61
|
-
const processor = new ProcessorBase(mockLogger, 2);
|
|
62
|
-
|
|
63
|
-
// Should not throw
|
|
64
|
-
await processor.process([]);
|
|
65
|
-
});
|
|
66
|
-
|
|
67
|
-
test("calls processItem for each item", async () => {
|
|
68
|
-
/** Test processor that tracks processed items */
|
|
69
|
-
class TestProcessor extends ProcessorBase {
|
|
70
|
-
/**
|
|
71
|
-
* Creates a test processor
|
|
72
|
-
* @param {object} logger - Logger instance
|
|
73
|
-
*/
|
|
74
|
-
constructor(logger) {
|
|
75
|
-
super(logger, 2);
|
|
76
|
-
this.processedItems = [];
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
/**
|
|
80
|
-
* Processes a single item
|
|
81
|
-
* @param {any} item - Item to process
|
|
82
|
-
* @returns {Promise<string>} Processed result
|
|
83
|
-
*/
|
|
84
|
-
async processItem(item) {
|
|
85
|
-
this.processedItems.push(item);
|
|
86
|
-
return `processed-${item}`;
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
const processor = new TestProcessor(mockLogger);
|
|
91
|
-
await processor.process(["a", "b", "c", "d"]);
|
|
92
|
-
|
|
93
|
-
assert.strictEqual(processor.processedItems.length, 4);
|
|
94
|
-
assert.deepStrictEqual(processor.processedItems, ["a", "b", "c", "d"]);
|
|
95
|
-
});
|
|
96
|
-
|
|
97
|
-
test("continues processing when individual items fail", async () => {
|
|
98
|
-
/** Test processor that simulates failures */
|
|
99
|
-
class TestProcessor extends ProcessorBase {
|
|
100
|
-
/**
|
|
101
|
-
* Creates a test processor
|
|
102
|
-
* @param {object} logger - Logger instance
|
|
103
|
-
*/
|
|
104
|
-
constructor(logger) {
|
|
105
|
-
super(logger, 3);
|
|
106
|
-
this.processedItems = [];
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
/**
|
|
110
|
-
* Processes a single item with failure simulation
|
|
111
|
-
* @param {any} item - Item to process
|
|
112
|
-
* @returns {Promise<string>} Processed result
|
|
113
|
-
*/
|
|
114
|
-
async processItem(item) {
|
|
115
|
-
if (item === "fail") {
|
|
116
|
-
throw new Error("Simulated failure");
|
|
117
|
-
}
|
|
118
|
-
this.processedItems.push(item);
|
|
119
|
-
return `processed-${item}`;
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
const processor = new TestProcessor(mockLogger);
|
|
124
|
-
await processor.process(["a", "fail", "b", "c"]);
|
|
125
|
-
|
|
126
|
-
// Should have processed all items except the failing one
|
|
127
|
-
assert.deepStrictEqual(processor.processedItems, ["a", "b", "c"]);
|
|
128
|
-
});
|
|
129
|
-
});
|
|
130
|
-
|
|
131
|
-
describe("processItem", () => {
|
|
132
|
-
test("throws error when not implemented", async () => {
|
|
133
|
-
const processor = new ProcessorBase(mockLogger, 2);
|
|
134
|
-
|
|
135
|
-
await assert.rejects(() => processor.processItem("item"), {
|
|
136
|
-
message: /processItem must be implemented by subclass/,
|
|
137
|
-
});
|
|
138
|
-
});
|
|
139
|
-
});
|
|
140
|
-
});
|
package/test/retry.test.js
DELETED
|
@@ -1,194 +0,0 @@
|
|
|
1
|
-
import { test, describe, beforeEach, mock } from "node:test";
|
|
2
|
-
import assert from "node:assert";
|
|
3
|
-
|
|
4
|
-
// Module under test
|
|
5
|
-
import { Retry } from "../retry.js";
|
|
6
|
-
|
|
7
|
-
describe("Retry", () => {
|
|
8
|
-
let retry;
|
|
9
|
-
|
|
10
|
-
beforeEach(() => {
|
|
11
|
-
// Use very short delay for testing to speed up retry tests
|
|
12
|
-
retry = new Retry({ delay: 1 });
|
|
13
|
-
});
|
|
14
|
-
|
|
15
|
-
test("creates retry instance with default config", () => {
|
|
16
|
-
const defaultRetry = new Retry();
|
|
17
|
-
assert.ok(defaultRetry instanceof Retry);
|
|
18
|
-
});
|
|
19
|
-
|
|
20
|
-
test("creates retry instance with custom config", () => {
|
|
21
|
-
const customRetry = new Retry({ retries: 5, delay: 500 });
|
|
22
|
-
assert.ok(customRetry instanceof Retry);
|
|
23
|
-
});
|
|
24
|
-
|
|
25
|
-
test("retry mechanism works with exhausted retries on 429", async () => {
|
|
26
|
-
const retryResponse = {
|
|
27
|
-
ok: false,
|
|
28
|
-
status: 429,
|
|
29
|
-
statusText: "Too Many Requests",
|
|
30
|
-
};
|
|
31
|
-
|
|
32
|
-
const mockFetch = mock.fn(() => Promise.resolve(retryResponse));
|
|
33
|
-
|
|
34
|
-
const response = await retry.execute(mockFetch);
|
|
35
|
-
|
|
36
|
-
// Should exhaust all retries and return the 429 response
|
|
37
|
-
assert.strictEqual(mockFetch.mock.callCount(), 11); // Initial + 10 retries
|
|
38
|
-
assert.strictEqual(response.status, 429);
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
test("retries on 502 Bad Gateway errors", async () => {
|
|
42
|
-
const retryResponse = {
|
|
43
|
-
ok: false,
|
|
44
|
-
status: 502,
|
|
45
|
-
statusText: "Bad Gateway",
|
|
46
|
-
};
|
|
47
|
-
const successResponse = {
|
|
48
|
-
ok: true,
|
|
49
|
-
status: 200,
|
|
50
|
-
json: mock.fn(() => Promise.resolve({ data: "success" })),
|
|
51
|
-
};
|
|
52
|
-
|
|
53
|
-
let callCount = 0;
|
|
54
|
-
const mockFetch = mock.fn(() => {
|
|
55
|
-
callCount++;
|
|
56
|
-
if (callCount === 1) {
|
|
57
|
-
return Promise.resolve(retryResponse);
|
|
58
|
-
}
|
|
59
|
-
return Promise.resolve(successResponse);
|
|
60
|
-
});
|
|
61
|
-
|
|
62
|
-
const result = await retry.execute(mockFetch);
|
|
63
|
-
|
|
64
|
-
// Should retry once and succeed
|
|
65
|
-
assert(mockFetch.mock.callCount() >= 2);
|
|
66
|
-
assert.strictEqual(result.ok, true);
|
|
67
|
-
});
|
|
68
|
-
|
|
69
|
-
test("retries on 503 Service Unavailable errors", async () => {
|
|
70
|
-
const retryResponse = {
|
|
71
|
-
ok: false,
|
|
72
|
-
status: 503,
|
|
73
|
-
statusText: "Service Unavailable",
|
|
74
|
-
};
|
|
75
|
-
const successResponse = {
|
|
76
|
-
ok: true,
|
|
77
|
-
status: 200,
|
|
78
|
-
json: mock.fn(() => Promise.resolve({ data: "success" })),
|
|
79
|
-
};
|
|
80
|
-
|
|
81
|
-
let callCount = 0;
|
|
82
|
-
const mockFetch = mock.fn(() => {
|
|
83
|
-
callCount++;
|
|
84
|
-
if (callCount === 1) {
|
|
85
|
-
return Promise.resolve(retryResponse);
|
|
86
|
-
}
|
|
87
|
-
return Promise.resolve(successResponse);
|
|
88
|
-
});
|
|
89
|
-
|
|
90
|
-
const result = await retry.execute(mockFetch);
|
|
91
|
-
|
|
92
|
-
// Should retry once and succeed
|
|
93
|
-
assert(mockFetch.mock.callCount() >= 2);
|
|
94
|
-
assert.strictEqual(result.ok, true);
|
|
95
|
-
});
|
|
96
|
-
|
|
97
|
-
test("retries on 504 Gateway Timeout errors", async () => {
|
|
98
|
-
const retryResponse = {
|
|
99
|
-
ok: false,
|
|
100
|
-
status: 504,
|
|
101
|
-
statusText: "Gateway Timeout",
|
|
102
|
-
};
|
|
103
|
-
const successResponse = {
|
|
104
|
-
ok: true,
|
|
105
|
-
status: 200,
|
|
106
|
-
json: mock.fn(() => Promise.resolve({ data: "success" })),
|
|
107
|
-
};
|
|
108
|
-
|
|
109
|
-
let callCount = 0;
|
|
110
|
-
const mockFetch = mock.fn(() => {
|
|
111
|
-
callCount++;
|
|
112
|
-
if (callCount <= 2) {
|
|
113
|
-
return Promise.resolve(retryResponse);
|
|
114
|
-
}
|
|
115
|
-
return Promise.resolve(successResponse);
|
|
116
|
-
});
|
|
117
|
-
|
|
118
|
-
const result = await retry.execute(mockFetch);
|
|
119
|
-
|
|
120
|
-
// Should retry twice and succeed
|
|
121
|
-
assert(mockFetch.mock.callCount() >= 3);
|
|
122
|
-
assert.strictEqual(result.ok, true);
|
|
123
|
-
});
|
|
124
|
-
|
|
125
|
-
test("retries on network errors", async () => {
|
|
126
|
-
const successResponse = {
|
|
127
|
-
ok: true,
|
|
128
|
-
status: 200,
|
|
129
|
-
json: mock.fn(() => Promise.resolve({ data: "success" })),
|
|
130
|
-
};
|
|
131
|
-
|
|
132
|
-
let callCount = 0;
|
|
133
|
-
const mockFetch = mock.fn(() => {
|
|
134
|
-
callCount++;
|
|
135
|
-
if (callCount === 1) {
|
|
136
|
-
return Promise.reject(new Error("Network error: Connection refused"));
|
|
137
|
-
}
|
|
138
|
-
return Promise.resolve(successResponse);
|
|
139
|
-
});
|
|
140
|
-
|
|
141
|
-
const result = await retry.execute(mockFetch);
|
|
142
|
-
|
|
143
|
-
// Should retry once and succeed
|
|
144
|
-
assert(mockFetch.mock.callCount() >= 2);
|
|
145
|
-
assert.strictEqual(result.ok, true);
|
|
146
|
-
});
|
|
147
|
-
|
|
148
|
-
test("non-retryable errors do not trigger retries", async () => {
|
|
149
|
-
const errorResponse = {
|
|
150
|
-
ok: false,
|
|
151
|
-
status: 400,
|
|
152
|
-
statusText: "Bad Request",
|
|
153
|
-
};
|
|
154
|
-
|
|
155
|
-
const mockFetch = mock.fn(() => Promise.resolve(errorResponse));
|
|
156
|
-
|
|
157
|
-
const response = await retry.execute(mockFetch);
|
|
158
|
-
|
|
159
|
-
// Should only make one call for non-retryable errors
|
|
160
|
-
assert.strictEqual(mockFetch.mock.callCount(), 1);
|
|
161
|
-
assert.strictEqual(response.status, 400);
|
|
162
|
-
});
|
|
163
|
-
|
|
164
|
-
test("500 Internal Server Error triggers retries", async () => {
|
|
165
|
-
const errorResponse = {
|
|
166
|
-
ok: false,
|
|
167
|
-
status: 500,
|
|
168
|
-
statusText: "Internal Server Error",
|
|
169
|
-
};
|
|
170
|
-
|
|
171
|
-
const mockFetch = mock.fn(() => Promise.resolve(errorResponse));
|
|
172
|
-
|
|
173
|
-
const response = await retry.execute(mockFetch);
|
|
174
|
-
|
|
175
|
-
// Should retry for 500 errors (retries + 1 initial attempt = 11 calls)
|
|
176
|
-
assert.strictEqual(mockFetch.mock.callCount(), 11);
|
|
177
|
-
assert.strictEqual(response.status, 500);
|
|
178
|
-
});
|
|
179
|
-
|
|
180
|
-
test("executes successfully without validation function", async () => {
|
|
181
|
-
const successResponse = {
|
|
182
|
-
ok: true,
|
|
183
|
-
status: 200,
|
|
184
|
-
json: mock.fn(() => Promise.resolve({ data: "success" })),
|
|
185
|
-
};
|
|
186
|
-
|
|
187
|
-
const mockFetch = mock.fn(() => Promise.resolve(successResponse));
|
|
188
|
-
|
|
189
|
-
const result = await retry.execute(mockFetch);
|
|
190
|
-
|
|
191
|
-
assert.strictEqual(mockFetch.mock.callCount(), 1);
|
|
192
|
-
assert.strictEqual(result.ok, true);
|
|
193
|
-
});
|
|
194
|
-
});
|
package/test/tokenizer.test.js
DELETED
|
@@ -1,123 +0,0 @@
|
|
|
1
|
-
import { test, describe, beforeEach } from "node:test";
|
|
2
|
-
import assert from "node:assert";
|
|
3
|
-
|
|
4
|
-
// Module under test
|
|
5
|
-
import { Tokenizer, ranks } from "../tokenizer.js";
|
|
6
|
-
import { countTokens, createTokenizer } from "../index.js";
|
|
7
|
-
|
|
8
|
-
describe("Tokenizer", () => {
|
|
9
|
-
describe("constructor", () => {
|
|
10
|
-
test("creates instance with ranks parameter", () => {
|
|
11
|
-
const tokenizer = new Tokenizer(ranks);
|
|
12
|
-
assert.ok(tokenizer instanceof Tokenizer);
|
|
13
|
-
});
|
|
14
|
-
|
|
15
|
-
test("creates instance without ranks parameter", () => {
|
|
16
|
-
const tokenizer = new Tokenizer();
|
|
17
|
-
assert.ok(tokenizer instanceof Tokenizer);
|
|
18
|
-
});
|
|
19
|
-
});
|
|
20
|
-
|
|
21
|
-
describe("encode", () => {
|
|
22
|
-
let tokenizer;
|
|
23
|
-
|
|
24
|
-
beforeEach(() => {
|
|
25
|
-
tokenizer = new Tokenizer(ranks);
|
|
26
|
-
});
|
|
27
|
-
|
|
28
|
-
test("handles empty string", () => {
|
|
29
|
-
const result = tokenizer.encode("");
|
|
30
|
-
assert.strictEqual(result.length, 0);
|
|
31
|
-
});
|
|
32
|
-
|
|
33
|
-
test("handles non-string input", () => {
|
|
34
|
-
const result = tokenizer.encode(null);
|
|
35
|
-
assert.strictEqual(result.length, 0);
|
|
36
|
-
});
|
|
37
|
-
|
|
38
|
-
test("encodes simple word", () => {
|
|
39
|
-
const result = tokenizer.encode("hello");
|
|
40
|
-
assert.ok(result.length >= 1);
|
|
41
|
-
assert.ok(Array.isArray(result));
|
|
42
|
-
});
|
|
43
|
-
|
|
44
|
-
test("encodes longer text", () => {
|
|
45
|
-
const shortText = "hello";
|
|
46
|
-
const longText = "hello world this is a longer piece of text";
|
|
47
|
-
|
|
48
|
-
const shortResult = tokenizer.encode(shortText);
|
|
49
|
-
const longResult = tokenizer.encode(longText);
|
|
50
|
-
|
|
51
|
-
assert.ok(longResult.length > shortResult.length);
|
|
52
|
-
});
|
|
53
|
-
|
|
54
|
-
test("handles whitespace-only text", () => {
|
|
55
|
-
const result = tokenizer.encode(" ");
|
|
56
|
-
assert.strictEqual(result.length, 0);
|
|
57
|
-
});
|
|
58
|
-
|
|
59
|
-
test("handles punctuation", () => {
|
|
60
|
-
const result = tokenizer.encode("Hello, world!");
|
|
61
|
-
assert.ok(result.length >= 3); // At least hello + comma + world + exclamation
|
|
62
|
-
});
|
|
63
|
-
|
|
64
|
-
test("handles mixed content", () => {
|
|
65
|
-
const result = tokenizer.encode("The year 2024 was great!");
|
|
66
|
-
assert.ok(result.length >= 5); // Multiple words and punctuation
|
|
67
|
-
});
|
|
68
|
-
|
|
69
|
-
test("provides reasonable approximation", () => {
|
|
70
|
-
// Test that the approximation is in a reasonable range
|
|
71
|
-
const text = "This is a test sentence with about ten words here.";
|
|
72
|
-
const result = tokenizer.encode(text);
|
|
73
|
-
|
|
74
|
-
// Should be roughly 10-15 tokens for this sentence
|
|
75
|
-
assert.ok(result.length >= 8);
|
|
76
|
-
assert.ok(result.length <= 20);
|
|
77
|
-
});
|
|
78
|
-
});
|
|
79
|
-
|
|
80
|
-
describe("decode", () => {
|
|
81
|
-
test("throws error when called", () => {
|
|
82
|
-
const tokenizer = new Tokenizer(ranks);
|
|
83
|
-
assert.throws(() => tokenizer.decode([1, 2, 3]), {
|
|
84
|
-
message: /decode\(\) not implemented/,
|
|
85
|
-
});
|
|
86
|
-
});
|
|
87
|
-
});
|
|
88
|
-
});
|
|
89
|
-
|
|
90
|
-
describe("Integration with libutil functions", () => {
|
|
91
|
-
describe("tokenizerFactory", () => {
|
|
92
|
-
test("creates Tokenizer instance", () => {
|
|
93
|
-
const tokenizer = createTokenizer();
|
|
94
|
-
assert.ok(tokenizer instanceof Tokenizer);
|
|
95
|
-
});
|
|
96
|
-
});
|
|
97
|
-
|
|
98
|
-
describe("countTokens", () => {
|
|
99
|
-
test("returns token count for text", () => {
|
|
100
|
-
const count = countTokens("hello world");
|
|
101
|
-
assert.ok(typeof count === "number");
|
|
102
|
-
assert.ok(count >= 1);
|
|
103
|
-
});
|
|
104
|
-
|
|
105
|
-
test("handles empty text", () => {
|
|
106
|
-
const count = countTokens("");
|
|
107
|
-
assert.strictEqual(count, 0);
|
|
108
|
-
});
|
|
109
|
-
|
|
110
|
-
test("uses provided tokenizer", () => {
|
|
111
|
-
const customTokenizer = new Tokenizer(ranks);
|
|
112
|
-
const count = countTokens("test", customTokenizer);
|
|
113
|
-
assert.ok(typeof count === "number");
|
|
114
|
-
assert.ok(count >= 1);
|
|
115
|
-
});
|
|
116
|
-
|
|
117
|
-
test("uses default tokenizer when none provided", () => {
|
|
118
|
-
const count = countTokens("test");
|
|
119
|
-
assert.ok(typeof count === "number");
|
|
120
|
-
assert.ok(count >= 1);
|
|
121
|
-
});
|
|
122
|
-
});
|
|
123
|
-
});
|