@maycombhq/sdk 0.0.0-stage → 2.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +177 -0
- package/dist/cache.d.ts +47 -0
- package/dist/cache.js +115 -0
- package/dist/cache.js.map +1 -0
- package/dist/fs-repo.d.ts +34 -0
- package/dist/fs-repo.js +120 -0
- package/dist/fs-repo.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -0
- package/dist/parse-cache.d.ts +97 -0
- package/dist/parse-cache.js +275 -0
- package/dist/parse-cache.js.map +1 -0
- package/dist/plugin.d.ts +89 -0
- package/dist/plugin.js +85 -0
- package/dist/plugin.js.map +1 -0
- package/dist/runner.d.ts +72 -0
- package/dist/runner.js +209 -0
- package/dist/runner.js.map +1 -0
- package/dist/sandbox.d.ts +80 -0
- package/dist/sandbox.js +443 -0
- package/dist/sandbox.js.map +1 -0
- package/package.json +29 -3
- package/README.md +0 -3
package/LICENSE
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
package/dist/cache.d.ts
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { type CatalogRecord } from "@maycombhq/core";
|
|
2
|
+
/**
|
|
3
|
+
* Scan cache. A scan of one repo by one extractor is a pure function of:
|
|
4
|
+
* the repo commit, the extractor id and version, the dialects in force and a hash of any
|
|
5
|
+
* configuration the extractor reads (for example dialect rule files). Same key, same
|
|
6
|
+
* records, so an unchanged repo is never scanned twice. See ADR 0010.
|
|
7
|
+
*/
|
|
8
|
+
export interface ScanCacheKey {
|
|
9
|
+
repo: string;
|
|
10
|
+
/** Full commit SHA of a clean checkout. Dirty trees are never cached. */
|
|
11
|
+
commit: string;
|
|
12
|
+
extractor: string;
|
|
13
|
+
version: string;
|
|
14
|
+
dialects: readonly string[];
|
|
15
|
+
/** Hash of configuration that changes the output (rule files, options). Empty when none. */
|
|
16
|
+
config?: string;
|
|
17
|
+
}
|
|
18
|
+
export interface CachedScan {
|
|
19
|
+
records: CatalogRecord[];
|
|
20
|
+
warnings: {
|
|
21
|
+
message: string;
|
|
22
|
+
path?: string;
|
|
23
|
+
}[];
|
|
24
|
+
/** Records the extractor emitted that failed contract C1; kept as a count for reports. */
|
|
25
|
+
rejected: number;
|
|
26
|
+
}
|
|
27
|
+
export interface ScanCache {
|
|
28
|
+
get(key: ScanCacheKey): Promise<CachedScan | undefined>;
|
|
29
|
+
put(key: ScanCacheKey, value: CachedScan): Promise<void>;
|
|
30
|
+
}
|
|
31
|
+
/** Stable hash of a cache key: dialect order doesn't matter, everything else does. */
|
|
32
|
+
export declare function cacheKeyHash(key: ScanCacheKey): string;
|
|
33
|
+
/** Hash configuration text (for example dialect rule files) into ScanCacheKey.config. */
|
|
34
|
+
export declare function configHash(parts: Record<string, string>): string;
|
|
35
|
+
export declare function memoryScanCache(): ScanCache & {
|
|
36
|
+
size(): number;
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* A scan cache on disk, normally `.index/scan-cache/` in the metarepo (git-ignored).
|
|
40
|
+
* One JSON file per key at `<repo>/<commit>/<extractor>@<version>.<hash>.json`.
|
|
41
|
+
* Writes are atomic (temp file, then rename). A file that fails its checksum or no longer
|
|
42
|
+
* passes contract C1 is treated as a miss, so a corrupt cache can only cost a rescan.
|
|
43
|
+
*/
|
|
44
|
+
export declare function fsScanCache(dir: string): ScanCache & {
|
|
45
|
+
/** Remove cached commits of a repo other than those listed. Returns how many were removed. */
|
|
46
|
+
prune(repo: string, keepCommits: readonly string[]): Promise<number>;
|
|
47
|
+
};
|
package/dist/cache.js
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { mkdir, readdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { parseRecord } from "@maycombhq/core";
|
|
5
|
+
const sha256 = (s) => createHash("sha256").update(s).digest("hex");
|
|
6
|
+
/** Stable hash of a cache key: dialect order doesn't matter, everything else does. */
|
|
7
|
+
export function cacheKeyHash(key) {
|
|
8
|
+
return sha256(JSON.stringify([
|
|
9
|
+
key.repo,
|
|
10
|
+
key.commit,
|
|
11
|
+
key.extractor,
|
|
12
|
+
key.version,
|
|
13
|
+
[...key.dialects].sort(),
|
|
14
|
+
key.config ?? "",
|
|
15
|
+
]));
|
|
16
|
+
}
|
|
17
|
+
/** Hash configuration text (for example dialect rule files) into ScanCacheKey.config. */
|
|
18
|
+
export function configHash(parts) {
|
|
19
|
+
const sorted = Object.keys(parts)
|
|
20
|
+
.sort()
|
|
21
|
+
.map((k) => [k, parts[k]]);
|
|
22
|
+
return sha256(JSON.stringify(sorted)).slice(0, 16);
|
|
23
|
+
}
|
|
24
|
+
export function memoryScanCache() {
|
|
25
|
+
const map = new Map();
|
|
26
|
+
return {
|
|
27
|
+
async get(key) {
|
|
28
|
+
const v = map.get(cacheKeyHash(key));
|
|
29
|
+
return v ? JSON.parse(v) : undefined;
|
|
30
|
+
},
|
|
31
|
+
async put(key, value) {
|
|
32
|
+
map.set(cacheKeyHash(key), JSON.stringify(value));
|
|
33
|
+
},
|
|
34
|
+
size: () => map.size,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
const SAFE = /^[A-Za-z0-9._@-]+$/;
|
|
38
|
+
/**
|
|
39
|
+
* A scan cache on disk, normally `.index/scan-cache/` in the metarepo (git-ignored).
|
|
40
|
+
* One JSON file per key at `<repo>/<commit>/<extractor>@<version>.<hash>.json`.
|
|
41
|
+
* Writes are atomic (temp file, then rename). A file that fails its checksum or no longer
|
|
42
|
+
* passes contract C1 is treated as a miss, so a corrupt cache can only cost a rescan.
|
|
43
|
+
*/
|
|
44
|
+
export function fsScanCache(dir) {
|
|
45
|
+
const fileFor = (key) => {
|
|
46
|
+
for (const part of [key.repo, key.commit, key.extractor, key.version]) {
|
|
47
|
+
if (!SAFE.test(part))
|
|
48
|
+
throw new Error(`unsafe cache key part: ${part}`);
|
|
49
|
+
}
|
|
50
|
+
const h = cacheKeyHash(key);
|
|
51
|
+
return {
|
|
52
|
+
h,
|
|
53
|
+
path: join(dir, key.repo, key.commit, `${key.extractor}@${key.version}.${h.slice(0, 16)}.json`),
|
|
54
|
+
};
|
|
55
|
+
};
|
|
56
|
+
return {
|
|
57
|
+
async get(key) {
|
|
58
|
+
const { h, path } = fileFor(key);
|
|
59
|
+
let file;
|
|
60
|
+
try {
|
|
61
|
+
file = JSON.parse(await readFile(path, "utf8"));
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
66
|
+
if (file.format !== 1 || file.keyHash !== h)
|
|
67
|
+
return undefined;
|
|
68
|
+
if (sha256(JSON.stringify(file.value)) !== file.checksum)
|
|
69
|
+
return undefined;
|
|
70
|
+
const records = [];
|
|
71
|
+
for (const r of file.value.records) {
|
|
72
|
+
const parsed = parseRecord(r);
|
|
73
|
+
if (!parsed.ok)
|
|
74
|
+
return undefined;
|
|
75
|
+
records.push(parsed.record);
|
|
76
|
+
}
|
|
77
|
+
return { ...file.value, records };
|
|
78
|
+
},
|
|
79
|
+
async put(key, value) {
|
|
80
|
+
const { h, path } = fileFor(key);
|
|
81
|
+
const file = {
|
|
82
|
+
format: 1,
|
|
83
|
+
key,
|
|
84
|
+
keyHash: h,
|
|
85
|
+
checksum: sha256(JSON.stringify(value)),
|
|
86
|
+
value,
|
|
87
|
+
};
|
|
88
|
+
await mkdir(join(path, ".."), { recursive: true });
|
|
89
|
+
const tmp = `${path}.${process.pid}.${Date.now()}.tmp`;
|
|
90
|
+
await writeFile(tmp, JSON.stringify(file), "utf8");
|
|
91
|
+
await rename(tmp, path);
|
|
92
|
+
},
|
|
93
|
+
async prune(repo, keepCommits) {
|
|
94
|
+
if (!SAFE.test(repo))
|
|
95
|
+
throw new Error(`unsafe repo name: ${repo}`);
|
|
96
|
+
const keep = new Set(keepCommits);
|
|
97
|
+
let removed = 0;
|
|
98
|
+
let commits;
|
|
99
|
+
try {
|
|
100
|
+
commits = await readdir(join(dir, repo));
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
return 0;
|
|
104
|
+
}
|
|
105
|
+
for (const c of commits) {
|
|
106
|
+
if (!keep.has(c)) {
|
|
107
|
+
await rm(join(dir, repo, c), { recursive: true, force: true });
|
|
108
|
+
removed++;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return removed;
|
|
112
|
+
},
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=cache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACnF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,WAAW,EAAsB,MAAM,iBAAiB,CAAC;AA+BlE,MAAM,MAAM,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAE3E,sFAAsF;AACtF,MAAM,UAAU,YAAY,CAAC,GAAiB;IAC5C,OAAO,MAAM,CACX,IAAI,CAAC,SAAS,CAAC;QACb,GAAG,CAAC,IAAI;QACR,GAAG,CAAC,MAAM;QACV,GAAG,CAAC,SAAS;QACb,GAAG,CAAC,OAAO;QACX,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE;QACxB,GAAG,CAAC,MAAM,IAAI,EAAE;KACjB,CAAC,CACH,CAAC;AACJ,CAAC;AAED,yFAAyF;AACzF,MAAM,UAAU,UAAU,CAAC,KAA6B;IACtD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;SAC9B,IAAI,EAAE;SACN,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,OAAO;QACL,KAAK,CAAC,GAAG,CAAC,GAAG;YACX,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YACrC,OAAO,CAAC,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAgB,CAAC,CAAC,CAAC,SAAS,CAAC;QACvD,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK;YAClB,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI;KACrB,CAAC;AACJ,CAAC;AAWD,MAAM,IAAI,GAAG,oBAAoB,CAAC;AAElC;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,GAAW;IAIrC,MAAM,OAAO,GAAG,CAAC,GAAiB,EAAE,EAAE;QACpC,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACtE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO;YACL,CAAC;YACD,IAAI,EAAE,IAAI,CACR,GAAG,EACH,GAAG,CAAC,IAAI,EACR,GAAG,CAAC,MAAM,EACV,GAAG,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CACzD;SACF,CAAC;IACJ,CAAC,CAAC;IAEF,OAAO;QACL,KAAK,CAAC,GAAG,CAAC,GAAG;YACX,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,IAAe,CAAC;YACpB,IAAI,CAAC;gBACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAc,CAAC;YAC/D,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,KAAK,CAAC;gBAAE,OAAO,SAAS,CAAC;YAC9D,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,CAAC,QAAQ;gBAAE,OAAO,SAAS,CAAC;YAC3E,MAAM,OAAO,GAAoB,EAAE,CAAC;YACpC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBACnC,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM,CAAC,EAAE;oBAAE,OAAO,SAAS,CAAC;gBACjC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;YACD,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC;QACpC,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK;YAClB,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;YACjC,MAAM,IAAI,GAAc;gBACtB,MAAM,EAAE,CAAC;gBACT,GAAG;gBACH,OAAO,EAAE,CAAC;gBACV,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACvC,KAAK;aACN,CAAC;YACF,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACnD,MAAM,GAAG,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC;YACvD,MAAM,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;YACnD,MAAM,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC1B,CAAC;QAED,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,WAAW;YAC3B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;YACnE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;YAClC,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,IAAI,OAAiB,CAAC;YACtB,IAAI,CAAC;gBACH,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC3C,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,CAAC,CAAC;YACX,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjB,MAAM,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC/D,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { type RepoHandle } from "./plugin.js";
|
|
2
|
+
/** Folders never scanned: VCS data, dependencies and build output. */
|
|
3
|
+
export declare const DEFAULT_IGNORES: readonly [".git", "node_modules", "target", "build", "dist", "out", ".gradle", ".idea", ".vscode", "__pycache__", ".venv"];
|
|
4
|
+
export interface FsRepoOptions {
|
|
5
|
+
/** Folder names skipped anywhere in the tree. Defaults to DEFAULT_IGNORES. */
|
|
6
|
+
ignore?: readonly string[];
|
|
7
|
+
/**
|
|
8
|
+
* Files larger than this (minified bundles, data dumps) are left out of listings and readText
|
|
9
|
+
* refuses them; skippedFiles() says which were left out. Default 5 MB.
|
|
10
|
+
*/
|
|
11
|
+
maxFileBytes?: number;
|
|
12
|
+
}
|
|
13
|
+
/** A RepoHandle over a folder, which can also say which files it left out for their size. */
|
|
14
|
+
export interface FsRepoHandle extends RepoHandle {
|
|
15
|
+
/** The size limit: files over it are left out of listings. */
|
|
16
|
+
readonly maxFileBytes: number;
|
|
17
|
+
/** Files a listing left out because they are over maxFileBytes, in path order. */
|
|
18
|
+
skippedFiles(): {
|
|
19
|
+
path: string;
|
|
20
|
+
bytes: number;
|
|
21
|
+
}[];
|
|
22
|
+
}
|
|
23
|
+
/** The warning for a file left out for its size: "left out: 5242881 bytes is over the 5 MB a file may be". */
|
|
24
|
+
export declare function tooLargeMessage(bytes: number, maxBytes: number): string;
|
|
25
|
+
export declare class RepoReadError extends Error {
|
|
26
|
+
readonly path: string;
|
|
27
|
+
constructor(message: string, path: string);
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* A read-only RepoHandle over a folder on disk (a base clone). Paths are repo-relative with
|
|
31
|
+
* forward slashes on every platform and listed in sorted order, so scans are deterministic.
|
|
32
|
+
* Symbolic links are never followed, and no path may leave the repo folder.
|
|
33
|
+
*/
|
|
34
|
+
export declare function fsRepo(name: string, dir: string, commit: string, options?: FsRepoOptions): FsRepoHandle;
|
package/dist/fs-repo.js
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { lstat, readdir, readFile, realpath, stat } from "node:fs/promises";
|
|
2
|
+
import { join, relative, resolve, sep } from "node:path";
|
|
3
|
+
import { globToRegExp } from "./plugin.js";
|
|
4
|
+
/** Folders never scanned: VCS data, dependencies and build output. */
|
|
5
|
+
export const DEFAULT_IGNORES = [
|
|
6
|
+
".git",
|
|
7
|
+
"node_modules",
|
|
8
|
+
"target",
|
|
9
|
+
"build",
|
|
10
|
+
"dist",
|
|
11
|
+
"out",
|
|
12
|
+
".gradle",
|
|
13
|
+
".idea",
|
|
14
|
+
".vscode",
|
|
15
|
+
"__pycache__",
|
|
16
|
+
".venv",
|
|
17
|
+
];
|
|
18
|
+
/** The warning for a file left out for its size: "left out: 5242881 bytes is over the 5 MB a file may be". */
|
|
19
|
+
export function tooLargeMessage(bytes, maxBytes) {
|
|
20
|
+
const mb = maxBytes / 1024 / 1024;
|
|
21
|
+
return `left out: ${bytes} bytes is over the ${Number.isInteger(mb) ? `${mb} MB` : `${maxBytes} bytes`} a file may be`;
|
|
22
|
+
}
|
|
23
|
+
export class RepoReadError extends Error {
|
|
24
|
+
path;
|
|
25
|
+
constructor(message, path) {
|
|
26
|
+
super(message);
|
|
27
|
+
this.path = path;
|
|
28
|
+
this.name = "RepoReadError";
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* A read-only RepoHandle over a folder on disk (a base clone). Paths are repo-relative with
|
|
33
|
+
* forward slashes on every platform and listed in sorted order, so scans are deterministic.
|
|
34
|
+
* Symbolic links are never followed, and no path may leave the repo folder.
|
|
35
|
+
*/
|
|
36
|
+
export function fsRepo(name, dir, commit, options = {}) {
|
|
37
|
+
const root = resolve(dir);
|
|
38
|
+
const ignore = new Set(options.ignore ?? DEFAULT_IGNORES);
|
|
39
|
+
const maxBytes = options.maxFileBytes ?? 5 * 1024 * 1024;
|
|
40
|
+
let cache;
|
|
41
|
+
/** Size of each file a listing matched, read once. */
|
|
42
|
+
const sizes = new Map();
|
|
43
|
+
const skipped = new Map();
|
|
44
|
+
async function walk(abs, out) {
|
|
45
|
+
const entries = await readdir(abs, { withFileTypes: true });
|
|
46
|
+
for (const e of entries) {
|
|
47
|
+
if (e.isSymbolicLink())
|
|
48
|
+
continue;
|
|
49
|
+
const full = join(abs, e.name);
|
|
50
|
+
if (e.isDirectory()) {
|
|
51
|
+
if (!ignore.has(e.name))
|
|
52
|
+
await walk(full, out);
|
|
53
|
+
}
|
|
54
|
+
else if (e.isFile()) {
|
|
55
|
+
out.push(relative(root, full).split(sep).join("/"));
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
async function allFiles() {
|
|
60
|
+
if (!cache) {
|
|
61
|
+
const out = [];
|
|
62
|
+
await walk(root, out);
|
|
63
|
+
cache = out.sort();
|
|
64
|
+
}
|
|
65
|
+
return cache;
|
|
66
|
+
}
|
|
67
|
+
async function safePath(path) {
|
|
68
|
+
if (path.startsWith("/") ||
|
|
69
|
+
/^[A-Za-z]:/.test(path) ||
|
|
70
|
+
path.includes("\\") ||
|
|
71
|
+
path.split("/").includes("..")) {
|
|
72
|
+
throw new RepoReadError(`path must be repo-relative with forward slashes: ${path}`, path);
|
|
73
|
+
}
|
|
74
|
+
const abs = join(root, ...path.split("/"));
|
|
75
|
+
const info = await lstat(abs).catch(() => undefined);
|
|
76
|
+
if (!info || !info.isFile())
|
|
77
|
+
throw new RepoReadError(`no such file: ${path}`, path);
|
|
78
|
+
if (info.isSymbolicLink())
|
|
79
|
+
throw new RepoReadError(`symbolic links are not read: ${path}`, path);
|
|
80
|
+
const real = await realpath(abs);
|
|
81
|
+
const realRoot = await realpath(root);
|
|
82
|
+
if (real !== realRoot && !real.startsWith(realRoot + sep)) {
|
|
83
|
+
throw new RepoReadError(`path leaves the repo: ${path}`, path);
|
|
84
|
+
}
|
|
85
|
+
if (info.size > maxBytes)
|
|
86
|
+
throw new RepoReadError(`file is larger than ${maxBytes} bytes: ${path}`, path);
|
|
87
|
+
return abs;
|
|
88
|
+
}
|
|
89
|
+
return {
|
|
90
|
+
name,
|
|
91
|
+
commit,
|
|
92
|
+
async *listFiles(globs) {
|
|
93
|
+
const matchers = globs.map(globToRegExp);
|
|
94
|
+
for (const p of await allFiles()) {
|
|
95
|
+
if (!matchers.some((m) => m.test(p)))
|
|
96
|
+
continue;
|
|
97
|
+
if (!sizes.has(p))
|
|
98
|
+
sizes.set(p, (await stat(join(root, ...p.split("/"))).catch(() => undefined))?.size ?? 0);
|
|
99
|
+
const bytes = sizes.get(p);
|
|
100
|
+
if (bytes > maxBytes) {
|
|
101
|
+
skipped.set(p, bytes);
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
yield p;
|
|
105
|
+
}
|
|
106
|
+
},
|
|
107
|
+
maxFileBytes: maxBytes,
|
|
108
|
+
skippedFiles() {
|
|
109
|
+
return [...skipped]
|
|
110
|
+
.sort(([a], [b]) => (a < b ? -1 : 1))
|
|
111
|
+
.map(([path, bytes]) => ({ path, bytes }));
|
|
112
|
+
},
|
|
113
|
+
async readText(path) {
|
|
114
|
+
const text = await readFile(await safePath(path), "utf8");
|
|
115
|
+
// Strip a UTF-8 byte-order mark so line numbers and parsers agree across editors.
|
|
116
|
+
return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
//# sourceMappingURL=fs-repo.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fs-repo.js","sourceRoot":"","sources":["../src/fs-repo.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC5E,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACzD,OAAO,EAAE,YAAY,EAAmB,MAAM,aAAa,CAAC;AAE5D,sEAAsE;AACtE,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,MAAM;IACN,cAAc;IACd,QAAQ;IACR,OAAO;IACP,MAAM;IACN,KAAK;IACL,SAAS;IACT,OAAO;IACP,SAAS;IACT,aAAa;IACb,OAAO;CACC,CAAC;AAoBX,8GAA8G;AAC9G,MAAM,UAAU,eAAe,CAAC,KAAa,EAAE,QAAgB;IAC7D,MAAM,EAAE,GAAG,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC;IAClC,OAAO,aAAa,KAAK,sBAAsB,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,QAAQ,QAAQ,gBAAgB,CAAC;AACzH,CAAC;AAED,MAAM,OAAO,aAAc,SAAQ,KAAK;IAG3B;IAFX,YACE,OAAe,EACN,IAAY;QAErB,KAAK,CAAC,OAAO,CAAC,CAAC;QAFN,SAAI,GAAJ,IAAI,CAAQ;QAGrB,IAAI,CAAC,IAAI,GAAG,eAAe,CAAC;IAC9B,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,UAAU,MAAM,CACpB,IAAY,EACZ,GAAW,EACX,MAAc,EACd,UAAyB,EAAE;IAE3B,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC1B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,eAAe,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;IACzD,IAAI,KAA2B,CAAC;IAChC,sDAAsD;IACtD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE1C,KAAK,UAAU,IAAI,CAAC,GAAW,EAAE,GAAa;QAC5C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,IAAI,CAAC,CAAC,cAAc,EAAE;gBAAE,SAAS;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YAC/B,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;oBAAE,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACjD,CAAC;iBAAM,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;gBACtB,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,UAAU,QAAQ;QACrB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,GAAG,GAAa,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACtB,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACrB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,UAAU,QAAQ,CAAC,IAAY;QAClC,IACE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YACpB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;YACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YACnB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAC9B,CAAC;YACD,MAAM,IAAI,aAAa,CAAC,oDAAoD,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;QAC5F,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3C,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAAE,MAAM,IAAI,aAAa,CAAC,iBAAiB,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;QACpF,IAAI,IAAI,CAAC,cAAc,EAAE;YACvB,MAAM,IAAI,aAAa,CAAC,gCAAgC,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,IAAI,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,GAAG,GAAG,CAAC,EAAE,CAAC;YAC1D,MAAM,IAAI,aAAa,CAAC,yBAAyB,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;QACjE,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,GAAG,QAAQ;YACtB,MAAM,IAAI,aAAa,CAAC,uBAAuB,QAAQ,WAAW,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;QAClF,OAAO,GAAG,CAAC;IACb,CAAC;IAED,OAAO;QACL,IAAI;QACJ,MAAM;QACN,KAAK,CAAC,CAAC,SAAS,CAAC,KAAK;YACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACzC,KAAK,MAAM,CAAC,IAAI,MAAM,QAAQ,EAAE,EAAE,CAAC;gBACjC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBAAE,SAAS;gBAC/C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;oBACf,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC;gBAC5F,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;gBAC5B,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;oBACrB,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;oBACtB,SAAS;gBACX,CAAC;gBACD,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;QACD,YAAY,EAAE,QAAQ;QACtB,YAAY;YACV,OAAO,CAAC,GAAG,OAAO,CAAC;iBAChB,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBACpC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QAC/C,CAAC;QACD,KAAK,CAAC,QAAQ,CAAC,IAAI;YACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;YAC1D,kFAAkF;YAClF,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9D,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
ADDED
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC"}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import type { RepoHandle } from "./plugin.js";
|
|
2
|
+
/**
|
|
3
|
+
* The parse cache (ADR 0067): what a file parses to, made once per repo scan and shared by every
|
|
4
|
+
* reader of that scan, built-in extractors, dialects and plugins alike. An entry is keyed by the
|
|
5
|
+
* reader's kind (what was made: "yaml", "java", "tree:java"), the file's path and its content, so a
|
|
6
|
+
* changed file or a different string under the same path (a rendered template) parses again.
|
|
7
|
+
*
|
|
8
|
+
* Values are deep-frozen before anyone sees them: a reader that changes what it was given fails
|
|
9
|
+
* at once instead of changing what the next reader reads. Readers that need to change a value
|
|
10
|
+
* copy it first. Memory is bounded twice over: plain values, and native values (tree-sitter
|
|
11
|
+
* trees, which live in the WebAssembly heap and must be freed), each under their own size
|
|
12
|
+
* bound, least recently used first out. A native value is freed when it leaves the cache, never
|
|
13
|
+
* while a reader holds it, and everything is freed when the scan ends.
|
|
14
|
+
*/
|
|
15
|
+
export interface ParseCache {
|
|
16
|
+
/** What `parse` makes of this text, parsed once; frozen unless `freeze: false`. */
|
|
17
|
+
get<T>(kind: string, path: string, text: string, parse: (text: string) => T, o?: GetOptions): T;
|
|
18
|
+
/** The same for a parser that is asynchronous; readers asking at once share one parse. */
|
|
19
|
+
getAsync<T>(kind: string, path: string, text: string, parse: (text: string) => Promise<T> | T, o?: GetOptions): Promise<T>;
|
|
20
|
+
/**
|
|
21
|
+
* Hand the value to `use` and keep it alive until `use` is done, for values that are freed
|
|
22
|
+
* (`dispose`): a syntax tree is never freed under a reader walking it. After the scan has
|
|
23
|
+
* ended, the value is made, used and freed at once.
|
|
24
|
+
*/
|
|
25
|
+
use<T, R>(kind: string, path: string, text: string, parse: (text: string) => Promise<T> | T, use: (value: T) => R | Promise<R>, o?: UseOptions<T>): Promise<R>;
|
|
26
|
+
/** A file's text, read once (`withParseCache` reads through this). */
|
|
27
|
+
text(path: string, read: () => Promise<string>): Promise<string>;
|
|
28
|
+
stats(): ParseCacheStats;
|
|
29
|
+
/** Free every value and keep nothing more: the end of the repo scan. Safe to call twice. */
|
|
30
|
+
dispose(): void;
|
|
31
|
+
}
|
|
32
|
+
export interface GetOptions {
|
|
33
|
+
/** Deep-freeze the value (default). Only values holding native handles pass false. */
|
|
34
|
+
freeze?: boolean;
|
|
35
|
+
/** "native" for values outside the JavaScript heap (syntax trees): a separate, smaller bound. */
|
|
36
|
+
pool?: "values" | "native";
|
|
37
|
+
/** What the value counts against its bound; the text's length when not given. */
|
|
38
|
+
weight?: number;
|
|
39
|
+
}
|
|
40
|
+
export interface UseOptions<T> extends GetOptions {
|
|
41
|
+
/** Free the value when it leaves the cache: `tree.delete()` for a syntax tree. */
|
|
42
|
+
dispose?: (value: T) => void;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* The bounds, in characters of the source each value was parsed from. Measured: a parsed value
|
|
46
|
+
* holds 10 (a JavaFile, an HCL file, flattened YAML) to 37 (YAML documents, a SourceFile) bytes of
|
|
47
|
+
* JavaScript heap per source character, and a syntax tree up to 36 bytes of WebAssembly heap,
|
|
48
|
+
* which never shrinks once grown. The defaults keep one repo scan's cache to about 300 MB of heap
|
|
49
|
+
* and 36 MB of WebAssembly memory at worst, for a 7 GB laptop.
|
|
50
|
+
*/
|
|
51
|
+
export interface ParseCacheLimits {
|
|
52
|
+
/** Plain values: the total length of the texts they were parsed from. Default 8M characters. */
|
|
53
|
+
maxChars?: number;
|
|
54
|
+
/** Plain values: how many. Default 20,000. */
|
|
55
|
+
maxEntries?: number;
|
|
56
|
+
/** Native values (syntax trees): total text length. Default 1M characters. */
|
|
57
|
+
maxNativeChars?: number;
|
|
58
|
+
/** Native values: how many. Default 2,000. */
|
|
59
|
+
maxNativeEntries?: number;
|
|
60
|
+
/** File texts kept by `text()`. Default 16M characters. */
|
|
61
|
+
maxTextChars?: number;
|
|
62
|
+
}
|
|
63
|
+
export interface ParseCacheStats {
|
|
64
|
+
hits: number;
|
|
65
|
+
misses: number;
|
|
66
|
+
evictions: number;
|
|
67
|
+
/** Plain values held now, and the length of the texts they came from. */
|
|
68
|
+
entries: number;
|
|
69
|
+
chars: number;
|
|
70
|
+
/** Native values held now, and their texts' length. */
|
|
71
|
+
nativeEntries: number;
|
|
72
|
+
nativeChars: number;
|
|
73
|
+
/** File texts held now; reads served from the cache and from the repo. */
|
|
74
|
+
texts: number;
|
|
75
|
+
textHits: number;
|
|
76
|
+
textReads: number;
|
|
77
|
+
/** Misses and hits by kind, for measuring which readers share. */
|
|
78
|
+
byKind: Record<string, {
|
|
79
|
+
hits: number;
|
|
80
|
+
misses: number;
|
|
81
|
+
}>;
|
|
82
|
+
}
|
|
83
|
+
export declare const DEFAULT_PARSE_CACHE_LIMITS: Required<ParseCacheLimits>;
|
|
84
|
+
/**
|
|
85
|
+
* Freeze a value and everything it holds, so no reader can change it: objects, arrays and class
|
|
86
|
+
* instances (their own properties; prototypes are shared and left alone), and Maps and Sets
|
|
87
|
+
* (their set, delete, clear and add throw). Typed arrays can't be frozen and are left as they are;
|
|
88
|
+
* functions are not walked into.
|
|
89
|
+
*/
|
|
90
|
+
export declare function deepFreeze<T>(value: T): T;
|
|
91
|
+
/** A parse cache for one repo scan. */
|
|
92
|
+
export declare function createParseCache(limits?: ParseCacheLimits): ParseCache;
|
|
93
|
+
/**
|
|
94
|
+
* The repo handle with a parse cache: readText reads each file once, and readers find the cache
|
|
95
|
+
* at `parseCache`. Everything else is the wrapped handle's (an fsRepo's skipped files too).
|
|
96
|
+
*/
|
|
97
|
+
export declare function withParseCache<R extends RepoHandle>(repo: R, cache: ParseCache): R;
|