@forwardimpact/libutil 0.1.70 → 0.1.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/fit-download-bundle.js +25 -2
- package/bin/fit-tiktoken.js +31 -7
- package/package.json +13 -2
- package/{finder.js → src/finder.js} +4 -1
- package/test/downloader.test.js +0 -223
- package/test/extractor.test.js +0 -338
- package/test/finder.test.js +0 -358
- package/test/fixtures/sample.tar.gz +0 -0
- package/test/fixtures/sample.zip +0 -0
- package/test/http.test.js +0 -93
- package/test/libutil.test.js +0 -35
- package/test/logger.test.js +0 -219
- package/test/processor.test.js +0 -140
- package/test/retry.test.js +0 -194
- package/test/tokenizer.test.js +0 -123
- package/test/wait.test.js +0 -109
- /package/{downloader.js → src/downloader.js} +0 -0
- /package/{extractor.js → src/extractor.js} +0 -0
- /package/{http.js → src/http.js} +0 -0
- /package/{index.js → src/index.js} +0 -0
- /package/{processor.js → src/processor.js} +0 -0
- /package/{retry.js → src/retry.js} +0 -0
- /package/{tokenizer.js → src/tokenizer.js} +0 -0
- /package/{wait.js → src/wait.js} +0 -0
|
@@ -1,19 +1,41 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
|
+
import { readFileSync } from "node:fs";
|
|
3
4
|
import { spawn } from "node:child_process";
|
|
5
|
+
import { createCli } from "@forwardimpact/libcli";
|
|
4
6
|
import { createScriptConfig } from "@forwardimpact/libconfig";
|
|
5
7
|
import { createStorage } from "@forwardimpact/libstorage";
|
|
6
8
|
import { createLogger } from "@forwardimpact/libtelemetry";
|
|
7
9
|
import { createBundleDownloader, execLine } from "@forwardimpact/libutil";
|
|
8
10
|
|
|
11
|
+
const { version: VERSION } = JSON.parse(
|
|
12
|
+
readFileSync(new URL("../package.json", import.meta.url), "utf8"),
|
|
13
|
+
);
|
|
14
|
+
|
|
15
|
+
const definition = {
|
|
16
|
+
name: "fit-download-bundle",
|
|
17
|
+
version: VERSION,
|
|
18
|
+
description: "Download generated code bundle from remote storage",
|
|
19
|
+
options: {
|
|
20
|
+
help: { type: "boolean", short: "h", description: "Show this help" },
|
|
21
|
+
version: { type: "boolean", description: "Show version" },
|
|
22
|
+
json: { type: "boolean", description: "Output help as JSON" },
|
|
23
|
+
},
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
const cli = createCli(definition);
|
|
27
|
+
const logger = createLogger("generated");
|
|
28
|
+
|
|
9
29
|
/**
|
|
10
30
|
* Downloads generated code bundle from remote storage.
|
|
11
31
|
* Used in containerized deployments to fetch pre-generated code.
|
|
12
32
|
* @returns {Promise<void>}
|
|
13
33
|
*/
|
|
14
34
|
async function main() {
|
|
35
|
+
const parsed = cli.parse(process.argv.slice(2));
|
|
36
|
+
if (!parsed) process.exit(0);
|
|
37
|
+
|
|
15
38
|
await createScriptConfig("download-bundle");
|
|
16
|
-
const logger = createLogger("generated");
|
|
17
39
|
const downloader = createBundleDownloader(createStorage, logger);
|
|
18
40
|
await downloader.download();
|
|
19
41
|
|
|
@@ -22,6 +44,7 @@ async function main() {
|
|
|
22
44
|
}
|
|
23
45
|
|
|
24
46
|
main().catch((error) => {
|
|
25
|
-
|
|
47
|
+
logger.exception("main", error);
|
|
48
|
+
cli.error(error.message);
|
|
26
49
|
process.exit(1);
|
|
27
50
|
});
|
package/bin/fit-tiktoken.js
CHANGED
|
@@ -1,14 +1,38 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { createCli } from "@forwardimpact/libcli";
|
|
4
|
+
import { createLogger } from "@forwardimpact/libtelemetry";
|
|
2
5
|
import { countTokens } from "@forwardimpact/libutil";
|
|
3
6
|
|
|
7
|
+
const { version: VERSION } = JSON.parse(
|
|
8
|
+
readFileSync(new URL("../package.json", import.meta.url), "utf8"),
|
|
9
|
+
);
|
|
10
|
+
|
|
11
|
+
const definition = {
|
|
12
|
+
name: "fit-tiktoken",
|
|
13
|
+
version: VERSION,
|
|
14
|
+
description: "Count tokens in text",
|
|
15
|
+
usage: "fit-tiktoken <text>\n echo 'text' | fit-tiktoken",
|
|
16
|
+
options: {
|
|
17
|
+
help: { type: "boolean", short: "h", description: "Show this help" },
|
|
18
|
+
version: { type: "boolean", description: "Show version" },
|
|
19
|
+
json: { type: "boolean", description: "Output help as JSON" },
|
|
20
|
+
},
|
|
21
|
+
examples: ['fit-tiktoken "hello world"', "echo 'hello world' | fit-tiktoken"],
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
const cli = createCli(definition);
|
|
25
|
+
const logger = createLogger("tiktoken");
|
|
26
|
+
|
|
4
27
|
/**
|
|
5
28
|
* Counts tokens in the provided text
|
|
6
|
-
* Usage: fit-tiktoken <text>
|
|
7
|
-
* echo "text" | fit-tiktoken
|
|
8
29
|
* @returns {Promise<void>}
|
|
9
30
|
*/
|
|
10
31
|
async function main() {
|
|
11
|
-
|
|
32
|
+
const parsed = cli.parse(process.argv.slice(2));
|
|
33
|
+
if (!parsed) process.exit(0);
|
|
34
|
+
|
|
35
|
+
let text = parsed.positionals.join(" ");
|
|
12
36
|
|
|
13
37
|
if (!text && !process.stdin.isTTY) {
|
|
14
38
|
const chunks = [];
|
|
@@ -19,15 +43,15 @@ async function main() {
|
|
|
19
43
|
}
|
|
20
44
|
|
|
21
45
|
if (!text) {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
process.exit(1);
|
|
46
|
+
cli.usageError("expected text argument or stdin input");
|
|
47
|
+
process.exit(2);
|
|
25
48
|
}
|
|
26
49
|
|
|
27
50
|
console.log(countTokens(text));
|
|
28
51
|
}
|
|
29
52
|
|
|
30
53
|
main().catch((error) => {
|
|
31
|
-
|
|
54
|
+
logger.exception("main", error);
|
|
55
|
+
cli.error(error.message);
|
|
32
56
|
process.exit(1);
|
|
33
57
|
});
|
package/package.json
CHANGED
|
@@ -1,15 +1,25 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@forwardimpact/libutil",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.73",
|
|
4
4
|
"description": "Utility functions and utilities for Guide",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "D. Olsson <hi@senzilla.io>",
|
|
7
7
|
"type": "module",
|
|
8
|
-
"main": "index.js",
|
|
8
|
+
"main": "./src/index.js",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": "./src/index.js",
|
|
11
|
+
"./bin/fit-download-bundle.js": "./bin/fit-download-bundle.js",
|
|
12
|
+
"./bin/fit-tiktoken.js": "./bin/fit-tiktoken.js"
|
|
13
|
+
},
|
|
9
14
|
"bin": {
|
|
10
15
|
"fit-download-bundle": "./bin/fit-download-bundle.js",
|
|
11
16
|
"fit-tiktoken": "./bin/fit-tiktoken.js"
|
|
12
17
|
},
|
|
18
|
+
"files": [
|
|
19
|
+
"src/**/*.js",
|
|
20
|
+
"bin/**/*.js",
|
|
21
|
+
"README.md"
|
|
22
|
+
],
|
|
13
23
|
"engines": {
|
|
14
24
|
"bun": ">=1.2.0",
|
|
15
25
|
"node": ">=18.0.0"
|
|
@@ -18,6 +28,7 @@
|
|
|
18
28
|
"test": "bun run node --test test/*.test.js"
|
|
19
29
|
},
|
|
20
30
|
"dependencies": {
|
|
31
|
+
"@forwardimpact/libcli": "^0.1.0",
|
|
21
32
|
"@forwardimpact/libtelemetry": "^0.1.22"
|
|
22
33
|
},
|
|
23
34
|
"devDependencies": {
|
|
@@ -114,7 +114,7 @@ export class Finder {
|
|
|
114
114
|
*/
|
|
115
115
|
findGeneratedPath(projectRoot, packageName) {
|
|
116
116
|
const packagePath = this.findPackagePath(projectRoot, packageName);
|
|
117
|
-
return path.join(packagePath, "generated");
|
|
117
|
+
return path.join(packagePath, "src", "generated");
|
|
118
118
|
}
|
|
119
119
|
|
|
120
120
|
/**
|
|
@@ -139,6 +139,9 @@ export class Finder {
|
|
|
139
139
|
// Target doesn't exist, which is fine
|
|
140
140
|
}
|
|
141
141
|
|
|
142
|
+
// Ensure the target's parent directory exists before symlinking
|
|
143
|
+
await fsAsync.mkdir(path.dirname(targetPath), { recursive: true });
|
|
144
|
+
|
|
142
145
|
// Create the symlink
|
|
143
146
|
await fsAsync.symlink(sourcePath, targetPath, "dir");
|
|
144
147
|
this.#logger.debug("Finder", "Created symlink", {
|
package/test/downloader.test.js
DELETED
|
@@ -1,223 +0,0 @@
|
|
|
1
|
-
import { strict as assert } from "node:assert";
|
|
2
|
-
import { test, describe, beforeEach, mock } from "node:test";
|
|
3
|
-
|
|
4
|
-
import { createSilentLogger } from "@forwardimpact/libharness";
|
|
5
|
-
|
|
6
|
-
import { BundleDownloader } from "../downloader.js";
|
|
7
|
-
|
|
8
|
-
describe("BundleDownloader", () => {
|
|
9
|
-
let mockStorageFactory;
|
|
10
|
-
let mockExtractor;
|
|
11
|
-
let mockLogger;
|
|
12
|
-
let mockFinder;
|
|
13
|
-
let mockProcess;
|
|
14
|
-
let mockLocalStorage;
|
|
15
|
-
let mockRemoteStorage;
|
|
16
|
-
|
|
17
|
-
beforeEach(() => {
|
|
18
|
-
mockLocalStorage = {
|
|
19
|
-
ensureBucket: async () => {},
|
|
20
|
-
put: async () => {},
|
|
21
|
-
delete: async () => {},
|
|
22
|
-
path: (key = ".") => `/local/path/${key}`,
|
|
23
|
-
};
|
|
24
|
-
|
|
25
|
-
mockRemoteStorage = {
|
|
26
|
-
exists: async () => true,
|
|
27
|
-
get: async () => Buffer.from("bundle data"),
|
|
28
|
-
};
|
|
29
|
-
|
|
30
|
-
mockStorageFactory = (prefix, type) => {
|
|
31
|
-
return type === "local" ? mockLocalStorage : mockRemoteStorage;
|
|
32
|
-
};
|
|
33
|
-
|
|
34
|
-
mockExtractor = {
|
|
35
|
-
extract: mock.fn(async () => {}),
|
|
36
|
-
};
|
|
37
|
-
mockLogger = createSilentLogger();
|
|
38
|
-
mockFinder = {
|
|
39
|
-
createPackageSymlinks: mock.fn(async () => {}),
|
|
40
|
-
};
|
|
41
|
-
mockProcess = {
|
|
42
|
-
env: { STORAGE_TYPE: "s3" },
|
|
43
|
-
};
|
|
44
|
-
});
|
|
45
|
-
|
|
46
|
-
test("constructor validates required dependencies", () => {
|
|
47
|
-
assert.throws(
|
|
48
|
-
() =>
|
|
49
|
-
new BundleDownloader(
|
|
50
|
-
null,
|
|
51
|
-
mockFinder,
|
|
52
|
-
mockLogger,
|
|
53
|
-
mockExtractor,
|
|
54
|
-
mockProcess,
|
|
55
|
-
),
|
|
56
|
-
/createStorageFn is required/,
|
|
57
|
-
);
|
|
58
|
-
|
|
59
|
-
assert.throws(
|
|
60
|
-
() =>
|
|
61
|
-
new BundleDownloader(
|
|
62
|
-
mockStorageFactory,
|
|
63
|
-
null,
|
|
64
|
-
mockLogger,
|
|
65
|
-
mockExtractor,
|
|
66
|
-
mockProcess,
|
|
67
|
-
),
|
|
68
|
-
/finder is required/,
|
|
69
|
-
);
|
|
70
|
-
|
|
71
|
-
assert.throws(
|
|
72
|
-
() =>
|
|
73
|
-
new BundleDownloader(
|
|
74
|
-
mockStorageFactory,
|
|
75
|
-
mockFinder,
|
|
76
|
-
null,
|
|
77
|
-
mockExtractor,
|
|
78
|
-
mockProcess,
|
|
79
|
-
),
|
|
80
|
-
/logger is required/,
|
|
81
|
-
);
|
|
82
|
-
|
|
83
|
-
assert.throws(
|
|
84
|
-
() =>
|
|
85
|
-
new BundleDownloader(
|
|
86
|
-
mockStorageFactory,
|
|
87
|
-
mockFinder,
|
|
88
|
-
mockLogger,
|
|
89
|
-
null,
|
|
90
|
-
mockProcess,
|
|
91
|
-
),
|
|
92
|
-
/extractor is required/,
|
|
93
|
-
);
|
|
94
|
-
|
|
95
|
-
assert.throws(
|
|
96
|
-
() =>
|
|
97
|
-
new BundleDownloader(
|
|
98
|
-
mockStorageFactory,
|
|
99
|
-
mockFinder,
|
|
100
|
-
mockLogger,
|
|
101
|
-
mockExtractor,
|
|
102
|
-
null,
|
|
103
|
-
),
|
|
104
|
-
/process is required/,
|
|
105
|
-
);
|
|
106
|
-
});
|
|
107
|
-
|
|
108
|
-
test("downloads and extracts bundle when it exists", async () => {
|
|
109
|
-
const operations = [];
|
|
110
|
-
mockLocalStorage.put = async (key, data) => {
|
|
111
|
-
operations.push({ op: "put", key, hasData: !!data });
|
|
112
|
-
};
|
|
113
|
-
mockLocalStorage.delete = async (key) => {
|
|
114
|
-
operations.push({ op: "delete", key });
|
|
115
|
-
};
|
|
116
|
-
mockExtractor.extract = async (sourcePath, targetPath) => {
|
|
117
|
-
operations.push({ op: "extract", sourcePath, targetPath });
|
|
118
|
-
};
|
|
119
|
-
|
|
120
|
-
const download = new BundleDownloader(
|
|
121
|
-
mockStorageFactory,
|
|
122
|
-
mockFinder,
|
|
123
|
-
mockLogger,
|
|
124
|
-
mockExtractor,
|
|
125
|
-
mockProcess,
|
|
126
|
-
);
|
|
127
|
-
await download.initialize();
|
|
128
|
-
await download.download();
|
|
129
|
-
|
|
130
|
-
assert.strictEqual(operations.length, 3);
|
|
131
|
-
assert.deepStrictEqual(operations[0], {
|
|
132
|
-
op: "put",
|
|
133
|
-
key: "bundle.tar.gz",
|
|
134
|
-
hasData: true,
|
|
135
|
-
});
|
|
136
|
-
assert.strictEqual(operations[1].op, "extract");
|
|
137
|
-
assert.strictEqual(operations[1].sourcePath, "/local/path/bundle.tar.gz");
|
|
138
|
-
assert.strictEqual(operations[1].targetPath, "/local/path/.");
|
|
139
|
-
assert.deepStrictEqual(operations[2], {
|
|
140
|
-
op: "delete",
|
|
141
|
-
key: "bundle.tar.gz",
|
|
142
|
-
});
|
|
143
|
-
});
|
|
144
|
-
|
|
145
|
-
test("throws error when bundle does not exist", async () => {
|
|
146
|
-
mockRemoteStorage.exists = async () => false;
|
|
147
|
-
|
|
148
|
-
const download = new BundleDownloader(
|
|
149
|
-
mockStorageFactory,
|
|
150
|
-
mockFinder,
|
|
151
|
-
mockLogger,
|
|
152
|
-
mockExtractor,
|
|
153
|
-
mockProcess,
|
|
154
|
-
);
|
|
155
|
-
await download.initialize();
|
|
156
|
-
|
|
157
|
-
await assert.rejects(() => download.download(), {
|
|
158
|
-
message: /Bundle not found/,
|
|
159
|
-
});
|
|
160
|
-
});
|
|
161
|
-
|
|
162
|
-
test("initializes storage instances correctly", async () => {
|
|
163
|
-
const ensureBucketCalls = [];
|
|
164
|
-
mockLocalStorage.ensureBucket = async () => {
|
|
165
|
-
ensureBucketCalls.push("generated");
|
|
166
|
-
};
|
|
167
|
-
|
|
168
|
-
const download = new BundleDownloader(
|
|
169
|
-
mockStorageFactory,
|
|
170
|
-
mockFinder,
|
|
171
|
-
mockLogger,
|
|
172
|
-
mockExtractor,
|
|
173
|
-
mockProcess,
|
|
174
|
-
);
|
|
175
|
-
await download.initialize();
|
|
176
|
-
|
|
177
|
-
assert.strictEqual(ensureBucketCalls.length, 1);
|
|
178
|
-
assert.strictEqual(ensureBucketCalls[0], "generated");
|
|
179
|
-
|
|
180
|
-
// Should have called createPackageSymlinks
|
|
181
|
-
assert.strictEqual(mockFinder.createPackageSymlinks.mock.calls.length, 1);
|
|
182
|
-
assert.strictEqual(
|
|
183
|
-
mockFinder.createPackageSymlinks.mock.calls[0].arguments[0],
|
|
184
|
-
"/local/path/.",
|
|
185
|
-
);
|
|
186
|
-
});
|
|
187
|
-
|
|
188
|
-
test("skips download when STORAGE_TYPE is local", async () => {
|
|
189
|
-
mockProcess.env.STORAGE_TYPE = "local";
|
|
190
|
-
mockRemoteStorage.exists = mock.fn(async () => true);
|
|
191
|
-
|
|
192
|
-
const download = new BundleDownloader(
|
|
193
|
-
mockStorageFactory,
|
|
194
|
-
mockFinder,
|
|
195
|
-
mockLogger,
|
|
196
|
-
mockExtractor,
|
|
197
|
-
mockProcess,
|
|
198
|
-
);
|
|
199
|
-
await download.initialize();
|
|
200
|
-
await download.download();
|
|
201
|
-
|
|
202
|
-
// Should not have called remote storage
|
|
203
|
-
assert.strictEqual(mockRemoteStorage.exists.mock.calls.length, 0);
|
|
204
|
-
});
|
|
205
|
-
|
|
206
|
-
test("downloads when STORAGE_TYPE is s3", async () => {
|
|
207
|
-
mockProcess.env.STORAGE_TYPE = "s3";
|
|
208
|
-
mockRemoteStorage.exists = mock.fn(async () => true);
|
|
209
|
-
|
|
210
|
-
const download = new BundleDownloader(
|
|
211
|
-
mockStorageFactory,
|
|
212
|
-
mockFinder,
|
|
213
|
-
mockLogger,
|
|
214
|
-
mockExtractor,
|
|
215
|
-
mockProcess,
|
|
216
|
-
);
|
|
217
|
-
await download.initialize();
|
|
218
|
-
await download.download();
|
|
219
|
-
|
|
220
|
-
// Should have called remote storage
|
|
221
|
-
assert.strictEqual(mockRemoteStorage.exists.mock.calls.length, 1);
|
|
222
|
-
});
|
|
223
|
-
});
|