@intellectual-frontiers/libraries 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +8 -2
- package/dist/affected.d.ts +28 -0
- package/dist/affected.js +250 -0
- package/dist/changes.d.ts +27 -0
- package/dist/changes.js +470 -0
- package/dist/check.d.ts +15 -0
- package/dist/check.js +198 -0
- package/dist/generate-ontology.d.ts +37 -0
- package/dist/generate-ontology.js +293 -0
- package/dist/generate-workflows.d.ts +12 -0
- package/dist/generate-workflows.js +102 -0
- package/dist/imports/nist-sp-800-53.import.d.ts +33 -0
- package/dist/imports/nist-sp-800-53.import.js +34 -0
- package/dist/imports/schema-org.import.d.ts +33 -0
- package/dist/imports/schema-org.import.js +31 -0
- package/dist/install-bun.d.ts +1 -0
- package/dist/install-bun.js +58 -0
- package/dist/libraries.d.ts +79 -0
- package/dist/libraries.js +386 -0
- package/dist/mod.d.ts +16 -0
- package/dist/mod.js +16 -0
- package/dist/newer.d.ts +27 -0
- package/dist/newer.js +147 -0
- package/dist/oscal-register.d.ts +65 -0
- package/dist/oscal-register.js +344 -0
- package/dist/pack-library.d.ts +47 -0
- package/dist/pack-library.js +498 -0
- package/dist/pins.d.ts +15 -0
- package/dist/pins.js +29 -0
- package/dist/publish-ontology.d.ts +160 -0
- package/dist/publish-ontology.js +227 -0
- package/dist/versions.d.ts +26 -0
- package/dist/versions.js +556 -0
- package/dist/workflows/definitions.d.ts +65 -0
- package/dist/workflows/definitions.js +614 -0
- package/dist/workflows/mod.d.ts +10 -0
- package/dist/workflows/mod.js +10 -0
- package/dist/workflows/model.d.ts +143 -0
- package/dist/workflows/model.js +81 -0
- package/dist/workflows/render.d.ts +9 -0
- package/dist/workflows/render.js +179 -0
- package/package.json +129 -4
- package/src/affected.ts +265 -0
- package/src/changes.ts +546 -0
- package/src/check.ts +223 -0
- package/src/generate-ontology.ts +350 -0
- package/src/generate-workflows.ts +103 -0
- package/src/imports/nist-sp-800-53.import.ts +36 -0
- package/src/imports/schema-org.import.ts +31 -0
- package/src/install-bun.ts +53 -0
- package/src/libraries.ts +385 -0
- package/src/mod.ts +16 -0
- package/src/newer.ts +154 -0
- package/src/oscal-register.ts +443 -0
- package/src/pack-library.ts +522 -0
- package/src/pins.ts +30 -0
- package/src/publish-ontology.ts +354 -0
- package/src/versions.ts +594 -0
- package/src/workflows/definitions.ts +847 -0
- package/src/workflows/mod.ts +30 -0
- package/src/workflows/model.ts +187 -0
- package/src/workflows/render.ts +176 -0
package/src/affected.ts
ADDED
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a change affects (commons 0027-commons-libraries FR-019 to FR-022): the files a library's checks read, taken from its package and its sources, and the libraries that a list of changed files
|
|
3
|
+
* affects, so that CI runs what changed and nothing else.
|
|
4
|
+
*
|
|
5
|
+
* bun commons/libraries/src/affected.ts --base origin/main the libraries a change since the merge base with origin/main affects, as {"include":[{"name","folder","verify"}]}
|
|
6
|
+
* bun commons/libraries/src/affected.ts --files a.ts,b/c.ts the same for a list of files (paths from the repository's root)
|
|
7
|
+
* bun commons/libraries/src/affected.ts --all every library
|
|
8
|
+
* bun commons/libraries/src/affected.ts --inputs data the folders and files the library `data` reads, one a line
|
|
9
|
+
* other flags: --repo DIR (the repository, the working folder by default)
|
|
10
|
+
*
|
|
11
|
+
* @remarks
|
|
12
|
+
* A library depends on another folder of commons where its `package.json` names it by path (`link:`, `file:` or `workspace:`), and what it reads of it is what its sources import from the package (the
|
|
13
|
+
* `exports` of the dependency say which file each import is), what its scripts run by a path to another folder (`bun ../libraries/src/check.ts`), and everything those files import by a relative path, followed
|
|
14
|
+
* file by file. An import that no entry of `exports` names takes the whole dependency. The library's inputs are also `commons/tools/**`, `.github/actions/**`, its own workflow and its own folder. A library is
|
|
15
|
+
* affected where a changed file is among its inputs, or where a library it depends on is affected. Nothing is run and nothing is written; the result is the same for the same files.
|
|
16
|
+
*
|
|
17
|
+
* @limits
|
|
18
|
+
* It reads sources with a pattern for `import`, `export ... from` and `import()`, which finds every file a source names (and at worst a file that a comment names, which only adds an input); a file that
|
|
19
|
+
* is read by a program through a path it builds at run time is not found, and a dependency that a library has outside commons is not followed. Where Git cannot say what changed, every library is affected.
|
|
20
|
+
*
|
|
21
|
+
* @example
|
|
22
|
+
* ```ts
|
|
23
|
+
* import { affectedBy } from "./affected.ts";
|
|
24
|
+
* affectedBy(".", ["commons/data/engine/api.ts"]).map((l) => l.name); // ["data"]
|
|
25
|
+
* ```
|
|
26
|
+
*
|
|
27
|
+
* Spec: 0001-libraries FR-003
|
|
28
|
+
*
|
|
29
|
+
* @module
|
|
30
|
+
*/
|
|
31
|
+
import process from "node:process";
|
|
32
|
+
import { existsSync, readDirSync, readTextFileSync } from "@intellectual-frontiers/system/fs";
|
|
33
|
+
import { dirname, join, normalize, resolve, toPosix } from "@intellectual-frontiers/system/path";
|
|
34
|
+
import { stdio } from "@intellectual-frontiers/system";
|
|
35
|
+
import { run } from "@intellectual-frontiers/system/proc";
|
|
36
|
+
import { type Library, readLibraries } from "./libraries.ts";
|
|
37
|
+
|
|
38
|
+
/** What a library reads: its folder, the folders and files it takes from others, what it depends on, and the files it imports one by one. */
|
|
39
|
+
export type Inputs = {
|
|
40
|
+
/** The paths of the library's inputs: `commons/<name>/` and each folder (`commons/ts/kernel/**`) or file the library takes from another. */
|
|
41
|
+
readonly paths: readonly string[];
|
|
42
|
+
/** The folders of commons that the library depends on through its package. */
|
|
43
|
+
readonly dependsOn: readonly string[];
|
|
44
|
+
/** Every file of another folder that it reads, from the repository's root, in order. */
|
|
45
|
+
readonly files: readonly string[];
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
/** The folders whose change is every library's change: the tools that check a library and the action that installs Bun. */
|
|
49
|
+
export const SHARED = ["commons/tools/**", ".github/actions/**"] as const;
|
|
50
|
+
|
|
51
|
+
const SOURCE = /\.(?:ts|mts|js|mjs)$/;
|
|
52
|
+
const SPECIFIER = /(?:\bimport|\bexport)\s+(?:type\s+)?(?:[^'";]*?\s+from\s+)?["']([^"']+)["']|\bimport\(\s*["']([^"']+)["']\s*\)/g;
|
|
53
|
+
|
|
54
|
+
const posix = (p: string): string => toPosix(normalize(p));
|
|
55
|
+
|
|
56
|
+
function objectOf(x: unknown): Readonly<Record<string, unknown>> {
|
|
57
|
+
return typeof x === "object" && x !== null && !Array.isArray(x) ? Object.fromEntries(Object.entries(x)) : {};
|
|
58
|
+
}
|
|
59
|
+
const stringOf = (x: unknown): string | undefined => (typeof x === "string" ? x : undefined);
|
|
60
|
+
|
|
61
|
+
/** A package.json as the parts this reads. */
|
|
62
|
+
type Pkg = {
|
|
63
|
+
readonly name: string | undefined;
|
|
64
|
+
readonly exports: ReadonlyMap<string, string>;
|
|
65
|
+
readonly dependencies: ReadonlyMap<string, string>;
|
|
66
|
+
readonly scripts: readonly string[];
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
function readPackage(dir: string): Pkg | undefined {
|
|
70
|
+
const file = join(dir, "package.json");
|
|
71
|
+
if (!existsSync(file)) return undefined;
|
|
72
|
+
const o = objectOf(JSON.parse(readTextFileSync(file)) as unknown);
|
|
73
|
+
const exports = new Map<string, string>();
|
|
74
|
+
for (const [k, v] of Object.entries(objectOf(o["exports"]))) {
|
|
75
|
+
const target =
|
|
76
|
+
stringOf(v) ??
|
|
77
|
+
Object.values(objectOf(v))
|
|
78
|
+
.map(stringOf)
|
|
79
|
+
.find((s) => s !== undefined);
|
|
80
|
+
if (target !== undefined) exports.set(k, target);
|
|
81
|
+
}
|
|
82
|
+
const dependencies = new Map<string, string>();
|
|
83
|
+
for (const field of ["dependencies", "devDependencies", "peerDependencies", "optionalDependencies"])
|
|
84
|
+
for (const [k, v] of Object.entries(objectOf(o[field]))) if (typeof v === "string") dependencies.set(k, v);
|
|
85
|
+
return {
|
|
86
|
+
name: stringOf(o["name"]),
|
|
87
|
+
exports,
|
|
88
|
+
dependencies,
|
|
89
|
+
scripts: Object.values(objectOf(o["scripts"])).flatMap((s) => (typeof s === "string" ? [s] : [])),
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** The files under a folder that are sources, from the repository's root (`node_modules` and `dist` are not looked into). */
|
|
94
|
+
function sourcesUnder(root: string, dir: string): readonly string[] {
|
|
95
|
+
const out: string[] = [];
|
|
96
|
+
const walk = (d: string): void => {
|
|
97
|
+
for (const e of readDirSync(join(root, d))) {
|
|
98
|
+
if (e.name === "node_modules" || e.name === "dist" || e.name.startsWith(".")) continue;
|
|
99
|
+
const p = `${d}/${e.name}`;
|
|
100
|
+
if (e.isDirectory) walk(p);
|
|
101
|
+
else if (e.isFile && SOURCE.test(e.name)) out.push(p);
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
walk(dir);
|
|
105
|
+
return out.sort();
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const specifiersOf = (text: string): readonly string[] =>
|
|
109
|
+
[...text.matchAll(SPECIFIER)].map((m) => m[1] ?? m[2] ?? "").filter((s) => s !== "");
|
|
110
|
+
|
|
111
|
+
/** The file a relative specifier names, from the repository's root: the path as written, or with `.ts`, `/mod.ts` or `/index.ts`. */
|
|
112
|
+
function resolveFile(root: string, from: string, specifier: string): string | undefined {
|
|
113
|
+
const base = posix(join(dirname(from), specifier));
|
|
114
|
+
if (base.startsWith("..")) return undefined;
|
|
115
|
+
for (const c of [base, `${base}.ts`, `${base}/mod.ts`, `${base}/index.ts`]) if (SOURCE.test(c) && existsSync(join(root, c))) return c;
|
|
116
|
+
return undefined;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Every file reached from the entries by relative imports, the entries included. */
|
|
120
|
+
function closure(root: string, entries: readonly string[]): ReadonlySet<string> {
|
|
121
|
+
const seen = new Set<string>();
|
|
122
|
+
const todo = [...entries];
|
|
123
|
+
for (let f = todo.pop(); f !== undefined; f = todo.pop()) {
|
|
124
|
+
if (seen.has(f) || !existsSync(join(root, f))) continue;
|
|
125
|
+
seen.add(f);
|
|
126
|
+
if (!SOURCE.test(f)) continue;
|
|
127
|
+
for (const s of specifiersOf(readTextFileSync(join(root, f)))) {
|
|
128
|
+
if (!s.startsWith(".")) continue;
|
|
129
|
+
const r = resolveFile(root, f, s);
|
|
130
|
+
if (r !== undefined) todo.push(r);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return seen;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** The path as a filter reads it: the folder of a file two levels under `commons/<folder>/` is `commons/<folder>/<sub>/**`, one level `commons/<folder>/**`, and a file at the top is itself. */
|
|
137
|
+
function folderOf(file: string): string {
|
|
138
|
+
const parts = file.split("/");
|
|
139
|
+
if (parts[0] === "commons" && parts.length >= 4) return `commons/${parts[1] ?? ""}/${parts[2] ?? ""}/**`;
|
|
140
|
+
if (parts[0] === "commons" && parts.length === 3) return `commons/${parts[1] ?? ""}/**`;
|
|
141
|
+
return file;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** What the library `name` reads (FR-020). */
|
|
145
|
+
export function inputsOf(repo: string, name: string): Inputs {
|
|
146
|
+
const root = repo.replace(/[\\/]$/, "");
|
|
147
|
+
const folder = `commons/${name}`;
|
|
148
|
+
const pkg = readPackage(join(root, folder));
|
|
149
|
+
const sources = sourcesUnder(root, folder);
|
|
150
|
+
const texts = sources.map((f) => ({ f, text: readTextFileSync(join(root, f)) }));
|
|
151
|
+
const entries = new Set<string>();
|
|
152
|
+
const whole = new Set<string>();
|
|
153
|
+
const dependsOn = new Set<string>();
|
|
154
|
+
// a dependency by path: its folder, what the library's sources import from it, and the entries it maps them to
|
|
155
|
+
for (const [dep, spec] of pkg?.dependencies ?? []) {
|
|
156
|
+
const m = /^(?:link|file|workspace):(.+)$/.exec(spec);
|
|
157
|
+
if (m?.[1] === undefined) continue;
|
|
158
|
+
const dir = posix(join(folder, m[1]));
|
|
159
|
+
if (dir.startsWith("..") || !existsSync(join(root, dir))) continue;
|
|
160
|
+
const depPkg = readPackage(join(root, dir));
|
|
161
|
+
dependsOn.add(dir.split("/")[1] ?? dir);
|
|
162
|
+
const imported = new Set<string>();
|
|
163
|
+
for (const { text } of texts)
|
|
164
|
+
for (const s of specifiersOf(text))
|
|
165
|
+
if (s === dep || s.startsWith(`${dep}/`)) imported.add(s === dep ? "." : `.${s.slice(dep.length)}`);
|
|
166
|
+
for (const key of imported) {
|
|
167
|
+
const target = depPkg?.exports.get(key);
|
|
168
|
+
if (target === undefined) whole.add(`${dir}/**`);
|
|
169
|
+
else entries.add(posix(join(dir, target)));
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
// a path to another folder in a script of the package: `bun ../libraries/src/check.ts`
|
|
173
|
+
for (const script of pkg?.scripts ?? [])
|
|
174
|
+
for (const m of script.matchAll(/(?:^|\s)(\.\.\/[\w./-]+)/g)) {
|
|
175
|
+
const p = posix(join(folder, m[1] ?? ""));
|
|
176
|
+
if (!p.startsWith("..") && existsSync(join(root, p)) && SOURCE.test(p)) entries.add(p);
|
|
177
|
+
}
|
|
178
|
+
const files = [...closure(root, [...entries])].filter((f) => !f.startsWith(`${folder}/`)).sort();
|
|
179
|
+
const paths = [`${folder}/**`, ...new Set([...files.map(folderOf), ...whole, ...SHARED, `.github/workflows/${name}.auto.yml`])].sort();
|
|
180
|
+
for (const f of files) {
|
|
181
|
+
const d = f.split("/")[1];
|
|
182
|
+
if (d !== undefined && d !== name && existsSync(join(root, "commons", d, "library.ts"))) dependsOn.add(d);
|
|
183
|
+
}
|
|
184
|
+
return { paths, dependsOn: [...dependsOn].sort(), files };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** Whether a path of the repository is among the paths of an input: `dir/**` is the folder and what is under it, anything else the file. */
|
|
188
|
+
export function matches(paths: readonly string[], file: string): boolean {
|
|
189
|
+
return paths.some((p) => (p.endsWith("/**") ? file.startsWith(p.slice(0, -2)) : p === file));
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** The libraries that a list of changed files affects (FR-021): those with a changed file among their inputs, and those that depend on one of these, in order of their names. */
|
|
193
|
+
export function affectedBy(repo: string, changed: readonly string[]): readonly (Library & { readonly folder: string })[] {
|
|
194
|
+
const root = repo.replace(/[\\/]$/, "");
|
|
195
|
+
const libs = readLibraries(root).flatMap((f) => ("library" in f ? [f.library] : []));
|
|
196
|
+
const inputs = new Map(libs.map((l) => [l.name, inputsOf(root, l.name)]));
|
|
197
|
+
const hit = new Set(libs.filter((l) => changed.some((c) => matches(inputs.get(l.name)?.paths ?? [], c))).map((l) => l.name));
|
|
198
|
+
for (let grew = true; grew;) {
|
|
199
|
+
grew = false;
|
|
200
|
+
for (const l of libs) {
|
|
201
|
+
if (!hit.has(l.name) && (inputs.get(l.name)?.dependsOn ?? []).some((d) => hit.has(d))) {
|
|
202
|
+
hit.add(l.name);
|
|
203
|
+
grew = true;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
return libs.filter((l) => hit.has(l.name)).map((l) => ({ ...l, folder: `commons/${l.name}` }));
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** The files changed since the merge base of `base` and the head, tracked files of the working tree included; undefined where Git cannot say. */
|
|
211
|
+
export async function changedSince(repo: string, base: string): Promise<readonly string[] | undefined> {
|
|
212
|
+
const root = resolve(repo);
|
|
213
|
+
try {
|
|
214
|
+
const mb = await run(["git", "-C", root, "merge-base", base, "HEAD"]);
|
|
215
|
+
if (mb.code !== 0) return undefined;
|
|
216
|
+
const d = await run(["git", "-C", root, "diff", "--name-only", mb.stdout.trim()]);
|
|
217
|
+
return d.code === 0 ? d.stdout.split("\n").filter((l) => l !== "") : undefined;
|
|
218
|
+
} catch {
|
|
219
|
+
return undefined;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/** The matrix a workflow reads: `{"include":[{"name","folder","verify"}]}`. */
|
|
224
|
+
export function matrixOf(libs: readonly (Library & { readonly folder: string })[]): string {
|
|
225
|
+
return JSON.stringify({ include: libs.map((l) => ({ name: l.name, folder: l.folder, verify: l.verify })) });
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** The command line. Returns the exit status: 0, or 2 for a usage error. */
|
|
229
|
+
export async function main(argv: readonly string[]): Promise<number> {
|
|
230
|
+
const arg = (flag: string): string | undefined => {
|
|
231
|
+
const at = argv.indexOf(flag);
|
|
232
|
+
return at >= 0 ? argv[at + 1] : undefined;
|
|
233
|
+
};
|
|
234
|
+
const repo = arg("--repo") ?? ".";
|
|
235
|
+
const inputs = arg("--inputs");
|
|
236
|
+
if (inputs !== undefined) {
|
|
237
|
+
const i = inputsOf(repo, inputs);
|
|
238
|
+
stdio.stdout.writeSync(`${[...i.paths, ...i.files.map((f) => ` ${f}`)].join("\n")}\n`);
|
|
239
|
+
return 0;
|
|
240
|
+
}
|
|
241
|
+
const everything = readLibraries(repo).flatMap((f) => ("library" in f ? [{ ...f.library, folder: `commons/${f.dir}` }] : []));
|
|
242
|
+
const files = arg("--files");
|
|
243
|
+
const base = arg("--base");
|
|
244
|
+
if (argv.includes("--all")) stdio.stdout.writeSync(`${matrixOf(everything)}\n`);
|
|
245
|
+
else if (files !== undefined)
|
|
246
|
+
stdio.stdout.writeSync(
|
|
247
|
+
`${matrixOf(
|
|
248
|
+
affectedBy(
|
|
249
|
+
repo,
|
|
250
|
+
files.split(",").filter((f) => f !== ""),
|
|
251
|
+
),
|
|
252
|
+
)}\n`,
|
|
253
|
+
);
|
|
254
|
+
else if (base !== undefined) {
|
|
255
|
+
const changed = await changedSince(repo, base);
|
|
256
|
+
if (changed === undefined) stdio.stderr.writeSync(`affected: git cannot say what changed since ${base}; every library is affected\n`);
|
|
257
|
+
stdio.stdout.writeSync(`${matrixOf(changed === undefined ? everything : affectedBy(repo, changed))}\n`);
|
|
258
|
+
} else {
|
|
259
|
+
stdio.stderr.writeSync("affected: give --base REF, --files a,b, --all or --inputs NAME\n");
|
|
260
|
+
return 2;
|
|
261
|
+
}
|
|
262
|
+
return 0;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
if (import.meta.main) process.exitCode = await main(process.argv.slice(2));
|