@vltpkg/security-archive 0.0.0-28 → 0.0.0-30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/browser.d.ts +1 -1
- package/dist/esm/browser.d.ts.map +1 -1
- package/dist/esm/browser.js +1 -0
- package/dist/esm/browser.js.map +1 -1
- package/dist/esm/index.d.ts +5 -5
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +57 -38
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/types.d.ts +2 -7
- package/dist/esm/types.d.ts.map +1 -1
- package/dist/esm/types.js.map +1 -1
- package/package.json +7 -7
package/dist/esm/browser.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../../src/browser.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EACpB,MAAM,YAAY,CAAA;AACnB,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AAE3C,
|
|
1
|
+
{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../../src/browser.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EACpB,MAAM,YAAY,CAAA;AACnB,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AAE3C,cAAc,YAAY,CAAA;AAsB1B;;GAEG;AACH,qBAAa,eACX,SAAQ,GAAG,CAAC,KAAK,EAAE,iBAAiB,CACpC,YAAW,mBAAmB;IAE9B;;OAEG;IACH,EAAE,UAAQ;IAEV;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO;CAU1B"}
|
package/dist/esm/browser.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { asDepID } from '@vltpkg/dep-id/browser';
|
|
2
2
|
import { error } from '@vltpkg/error-cause';
|
|
3
3
|
import { asPackageReportData } from "./types.js";
|
|
4
|
+
export * from "./types.js";
|
|
4
5
|
const isObj = (o) => !!o && typeof o === 'object';
|
|
5
6
|
const isSecurityArchiveJSON = (json) => isObj(json) &&
|
|
6
7
|
Object.entries(json).every(([key, value]) => typeof key === 'string' && isObj(value));
|
package/dist/esm/browser.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser.js","sourceRoot":"","sources":["../../src/browser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAA;AAChD,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAC3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;
|
|
1
|
+
{"version":3,"file":"browser.js","sourceRoot":"","sources":["../../src/browser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAA;AAChD,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAC3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAQhD,cAAc,YAAY,CAAA;AAE1B,MAAM,KAAK,GAAG,CAAC,CAAU,EAAgC,EAAE,CACzD,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAA;AAE9B,MAAM,qBAAqB,GAAG,CAC5B,IAAa,EACsB,EAAE,CACrC,KAAK,CAAC,IAAI,CAAC;IACX,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CACxB,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,CAC1D,CAAA;AAEH,MAAM,qBAAqB,GAAG,CAC5B,IAAa,EACc,EAAE;IAC7B,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAC/D,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC,CAAA;AAED;;GAEG;AACH,MAAM,OAAO,eACX,SAAQ,GAA6B;IAGrC;;OAEG;IACH,EAAE,GAAG,KAAK,CAAA;IAEV;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,IAAa;QACvB,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO,SAAS,CAAA;QAExC,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAA;QACrC,MAAM,IAAI,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAA;QACxC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAA;QACvD,CAAC;QACD,OAAO,OAAO,CAAA;IAChB,CAAC;CACF","sourcesContent":["import { asDepID } from '@vltpkg/dep-id/browser'\nimport { error } from '@vltpkg/error-cause'\nimport { asPackageReportData } from './types.ts'\nimport type { JSONField } from '@vltpkg/types'\nimport type {\n PackageReportData,\n SecurityArchiveLike,\n} from './types.ts'\nimport type { DepID } from '@vltpkg/dep-id'\n\nexport * from './types.ts'\n\nconst isObj = (o: unknown): o is Record<string, unknown> =>\n !!o && typeof o === 'object'\n\nconst isSecurityArchiveJSON = (\n json: unknown,\n): json is Record<string, JSONField> =>\n isObj(json) &&\n Object.entries(json).every(\n ([key, value]) => typeof key === 'string' && isObj(value),\n )\n\nconst asSecurityArchiveJSON = (\n json: unknown,\n): Record<string, JSONField> => {\n if (!isSecurityArchiveJSON(json)) {\n throw error('Invalid security archive JSON', { found: json })\n }\n return json\n}\n\n/**\n * A database of security information for given packages in a graph.\n */\nexport class SecurityArchive\n extends Map<DepID, PackageReportData>\n implements SecurityArchiveLike\n{\n /**\n * Whether the security archive is valid.\n */\n ok = false\n\n /**\n * Loads a security archive from a valid JSON dump.\n */\n static load(dump: unknown) {\n if (dump === undefined) return undefined\n\n const archive = new SecurityArchive()\n const json = asSecurityArchiveJSON(dump)\n for (const [key, value] of Object.entries(json)) {\n archive.set(asDepID(key), asPackageReportData(value))\n }\n return archive\n }\n}\n"]}
|
package/dist/esm/index.d.ts
CHANGED
|
@@ -40,7 +40,7 @@ export type SecurityArchiveOptions = LRUCache.OptionsBase<DepID, PackageReportDa
|
|
|
40
40
|
};
|
|
41
41
|
export declare const version: string;
|
|
42
42
|
/**
|
|
43
|
-
* A database of security information for given packages
|
|
43
|
+
* A database of security information for given packages from a list of nodes.
|
|
44
44
|
*
|
|
45
45
|
* Using the SecurityArchive.refresh() method will update the local cache
|
|
46
46
|
* with information from the socket.dev APIs or load from the local storage
|
|
@@ -50,8 +50,8 @@ export declare const version: string;
|
|
|
50
50
|
export declare class SecurityArchive extends LRUCache<DepID, PackageReportData> implements SecurityArchiveLike {
|
|
51
51
|
#private;
|
|
52
52
|
/**
|
|
53
|
-
* True if the refresh process was successful and report data
|
|
54
|
-
*
|
|
53
|
+
* True if the refresh process was successful and report data is available
|
|
54
|
+
* for all public registry packages from the initial list of nodes.
|
|
55
55
|
*/
|
|
56
56
|
ok: boolean;
|
|
57
57
|
/**
|
|
@@ -68,13 +68,13 @@ export declare class SecurityArchive extends LRUCache<DepID, PackageReportData>
|
|
|
68
68
|
static get defaultTtl(): number;
|
|
69
69
|
constructor(options?: SecurityArchiveOptions);
|
|
70
70
|
/**
|
|
71
|
-
* Starts the security archive by providing
|
|
71
|
+
* Starts the security archive by providing an array of {@link NodeLike} instances,
|
|
72
72
|
* its registry-based nodes are going to be used as valid potential entries.
|
|
73
73
|
*
|
|
74
74
|
* Any entry that is missing from the persisted cached values are going
|
|
75
75
|
* to be requested in a batch-request to the remote socket.dev API.
|
|
76
76
|
*/
|
|
77
|
-
refresh({
|
|
77
|
+
refresh({ nodes }: SecurityArchiveRefreshOptions): Promise<void>;
|
|
78
78
|
/**
|
|
79
79
|
* Outputs the current in-memory cache as a JSON object.
|
|
80
80
|
*/
|
package/dist/esm/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAOpC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAOpC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AAE3C,OAAO,KAAK,EACV,iBAAiB,EACjB,mBAAmB,EACnB,6BAA6B,EAC9B,MAAM,YAAY,CAAA;AAEnB,cAAc,YAAY,CAAA;AAM1B,eAAO,MAAM,qBAAqB,gCAAgC,CAAA;AAElE,MAAM,MAAM,gBAAgB,GAAG;IAC7B,SAAS,CAAC,EAAE,WAAW,CAAA;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE;QACL,OAAO,EAAE,MAAM,CAAA;QACf,OAAO,EAAE,MAAM,CAAA;QACf,WAAW,EAAE,MAAM,CAAA;QACnB,OAAO,EAAE,MAAM,CAAA;QACf,WAAW,EAAE,MAAM,CAAA;QACnB,aAAa,EAAE,MAAM,CAAA;KACtB,CAAA;CACF,CAAA;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACZ,CAAA;AAED,MAAM,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;AAE3D,MAAM,MAAM,sBAAsB,GAAG,QAAQ,CAAC,WAAW,CACvD,KAAK,EACL,iBAAiB,EACjB,OAAO,CACR,GAAG;IACF;;OAEG;IACH,WAAW,CAAC,EAAE,SAAS,CAAA;IACvB;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB,CAAA;AAwBD,eAAO,MAAQ,OAAO,QAKrB,CAAA;AAED;;;;;;;GAOG;AACH,qBAAa,eACX,SAAQ,QAAQ,CAAC,KAAK,EAAE,iBAAiB,CACzC,YAAW,mBAAmB;;IAS9B;;;OAGG;IACH,EAAE,UAAQ;IAEV;;OAEG;WACU,KAAK,CAChB,OAAO,EAAE,sBAAsB,GAAG,6BAA6B;IAOjE;;OAEG;IACH,MAAM,KAAK,UAAU,WAEpB;IAED;;OAEG;IACH,MAAM,KAAK,UAAU,WAEpB;gBAEW,OAAO,GAAE,sBAA2B;IA6QhD;;;;;;OAMG;IACG,OAAO,CAAC,EAAE,KAAK,EAAE,EAAE,6BAA6B;IA4DtD;;OAEG;IACH,MAAM;CAOP"}
|
package/dist/esm/index.js
CHANGED
|
@@ -17,7 +17,9 @@ export const targetSecurityRegisty = 'https://registry.npmjs.org/';
|
|
|
17
17
|
* socket.dev API. This function checks if a given depID is pointing
|
|
18
18
|
* to the public npm registry and returns `true` if it does.
|
|
19
19
|
*/
|
|
20
|
-
const usesTargetRegistry = (
|
|
20
|
+
const usesTargetRegistry = (node) => {
|
|
21
|
+
const depID = node.id;
|
|
22
|
+
const specOptions = node.options;
|
|
21
23
|
const [nodeType, nodeRegistry] = splitDepID(depID);
|
|
22
24
|
if (nodeType !== 'registry') {
|
|
23
25
|
return false;
|
|
@@ -30,17 +32,7 @@ const usesTargetRegistry = (depID, specOptions) => {
|
|
|
30
32
|
// Loads the version number to be used in the User-Agent header
|
|
31
33
|
export const { version } = loadPackageJson(import.meta.filename, process.env.__VLT_INTERNAL_CLI_PACKAGE_JSON);
|
|
32
34
|
/**
|
|
33
|
-
*
|
|
34
|
-
*/
|
|
35
|
-
const retrieveNodeFromGraph = (graph, name, version) => {
|
|
36
|
-
for (const node of graph.nodesByName.get(name) ?? []) {
|
|
37
|
-
if (node.version === version) {
|
|
38
|
-
return node;
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
};
|
|
42
|
-
/**
|
|
43
|
-
* A database of security information for given packages in a graph.
|
|
35
|
+
* A database of security information for given packages from a list of nodes.
|
|
44
36
|
*
|
|
45
37
|
* Using the SecurityArchive.refresh() method will update the local cache
|
|
46
38
|
* with information from the socket.dev APIs or load from the local storage
|
|
@@ -52,9 +44,11 @@ export class SecurityArchive extends LRUCache {
|
|
|
52
44
|
#pUpdateExpired;
|
|
53
45
|
#path;
|
|
54
46
|
#retries;
|
|
47
|
+
#nodesByName = new Map();
|
|
48
|
+
#nodesByID = new Map();
|
|
55
49
|
/**
|
|
56
|
-
* True if the refresh process was successful and report data
|
|
57
|
-
*
|
|
50
|
+
* True if the refresh process was successful and report data is available
|
|
51
|
+
* for all public registry packages from the initial list of nodes.
|
|
58
52
|
*/
|
|
59
53
|
ok = false;
|
|
60
54
|
/**
|
|
@@ -88,6 +82,16 @@ export class SecurityArchive extends LRUCache {
|
|
|
88
82
|
options.path ?? new XDG('vlt').cache('security-archive.db');
|
|
89
83
|
this.#retries = options.retries ?? 3;
|
|
90
84
|
}
|
|
85
|
+
/**
|
|
86
|
+
* Retrieves a node using its name and version.
|
|
87
|
+
*/
|
|
88
|
+
#retrieveNodeByNameVersion(name, version) {
|
|
89
|
+
for (const node of this.#nodesByName.get(name) ?? []) {
|
|
90
|
+
if (node.version === version) {
|
|
91
|
+
return node;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
91
95
|
/**
|
|
92
96
|
* Opens a connection to the database at the given path.
|
|
93
97
|
*/
|
|
@@ -108,9 +112,9 @@ export class SecurityArchive extends LRUCache {
|
|
|
108
112
|
/**
|
|
109
113
|
* Loads all valid items found in the database into the in-memory cache.
|
|
110
114
|
*/
|
|
111
|
-
#loadItemsFromDatabase(db,
|
|
115
|
+
#loadItemsFromDatabase(db, nodes) {
|
|
112
116
|
const depIDs = [];
|
|
113
|
-
for (const node of
|
|
117
|
+
for (const node of nodes) {
|
|
114
118
|
depIDs.push(`'${baseDepID(node.id)}'`);
|
|
115
119
|
}
|
|
116
120
|
// retrieves from the db packages using their dep-id reference
|
|
@@ -143,39 +147,43 @@ export class SecurityArchive extends LRUCache {
|
|
|
143
147
|
// cli commands that make usage of the security-archive, e.g: vlt ls,
|
|
144
148
|
// vlt query to not hang while waiting for stale-while-revalidate
|
|
145
149
|
// request to finish and close the db connection
|
|
146
|
-
this.#pUpdateExpired = this.#updateExpired(db
|
|
150
|
+
this.#pUpdateExpired = this.#updateExpired(db);
|
|
147
151
|
}
|
|
148
152
|
/**
|
|
149
153
|
* Updates the database with renewed entries for expired packages.
|
|
150
154
|
*/
|
|
151
|
-
async #updateExpired(db
|
|
155
|
+
async #updateExpired(db) {
|
|
152
156
|
const expiredQueue = new Set();
|
|
153
157
|
for (const depID of this.#expired) {
|
|
154
|
-
const node =
|
|
155
|
-
/* c8 ignore
|
|
156
|
-
if (!node)
|
|
158
|
+
const node = this.#nodesByID.get(baseDepID(depID));
|
|
159
|
+
/* c8 ignore start */
|
|
160
|
+
if (!node?.version) {
|
|
161
|
+
// skip any missing node or nodes without a version, this
|
|
162
|
+
// should not happen, but some optional dependencies might
|
|
163
|
+
// end up in this state when reading from the lockfile
|
|
157
164
|
continue;
|
|
165
|
+
}
|
|
166
|
+
/* c8 ignore stop */
|
|
158
167
|
const purl = `pkg:npm/${node.name}@${node.version}`;
|
|
159
168
|
expiredQueue.add({ purl });
|
|
160
169
|
}
|
|
161
170
|
if (expiredQueue.size > 0) {
|
|
162
171
|
const res = await pRetry(() => this.#retrieveRemoteData(expiredQueue), { retries: this.#retries });
|
|
163
|
-
const ids = this.#loadFromNDJSON(res
|
|
172
|
+
const ids = this.#loadFromNDJSON(res);
|
|
164
173
|
this.#storeNewItemsToDatabase(db, ids);
|
|
165
174
|
}
|
|
166
175
|
}
|
|
167
176
|
/**
|
|
168
177
|
* Queues up all required packages that are missing from the archive.
|
|
169
178
|
*/
|
|
170
|
-
#queueUpRequiredPackages(
|
|
179
|
+
#queueUpRequiredPackages() {
|
|
171
180
|
const queue = new Set();
|
|
172
|
-
const
|
|
173
|
-
for (const node of nodes) {
|
|
181
|
+
for (const node of this.#nodesByID.values()) {
|
|
174
182
|
const normalizedId = baseDepID(node.id);
|
|
175
183
|
// only queue up valid public registry
|
|
176
184
|
// references that are missing from the archive
|
|
177
185
|
// also skips any pkg marked as expired
|
|
178
|
-
if (usesTargetRegistry(node
|
|
186
|
+
if (usesTargetRegistry(node) &&
|
|
179
187
|
!this.has(normalizedId) &&
|
|
180
188
|
!this.#expired.has(normalizedId)) {
|
|
181
189
|
const purl = `pkg:npm/${node.name}@${node.version}`;
|
|
@@ -215,7 +223,7 @@ export class SecurityArchive extends LRUCache {
|
|
|
215
223
|
* Parses and load NDJSON data into the in-memory cache.
|
|
216
224
|
* Returns a set of dep ids that were successfully loaded.
|
|
217
225
|
*/
|
|
218
|
-
#loadFromNDJSON(str
|
|
226
|
+
#loadFromNDJSON(str) {
|
|
219
227
|
// builds a set of dep ids that were successfully fetched and loaded
|
|
220
228
|
const fetchedDepIDs = new Set();
|
|
221
229
|
const json = str.split('}\n');
|
|
@@ -225,7 +233,7 @@ export class SecurityArchive extends LRUCache {
|
|
|
225
233
|
const data = JSON.parse(line + '}');
|
|
226
234
|
const scope = data.namespace ? `${data.namespace}/` : '';
|
|
227
235
|
const name = `${scope}${data.name}`;
|
|
228
|
-
const node =
|
|
236
|
+
const node = this.#retrieveNodeByNameVersion(name, data.version);
|
|
229
237
|
if (node) {
|
|
230
238
|
const normalizedId = baseDepID(node.id);
|
|
231
239
|
fetchedDepIDs.add(baseDepID(node.id));
|
|
@@ -279,12 +287,12 @@ export class SecurityArchive extends LRUCache {
|
|
|
279
287
|
}
|
|
280
288
|
}
|
|
281
289
|
/**
|
|
282
|
-
* Validates that all public-registry packages in the
|
|
290
|
+
* Validates that all public-registry packages in the nodes
|
|
283
291
|
* have a valid report data in the current in-memory cache.
|
|
284
292
|
*/
|
|
285
|
-
#validateReportData(
|
|
286
|
-
for (const node of
|
|
287
|
-
if (usesTargetRegistry(node
|
|
293
|
+
#validateReportData() {
|
|
294
|
+
for (const node of this.#nodesByID.values()) {
|
|
295
|
+
if (usesTargetRegistry(node)) {
|
|
288
296
|
if (!this.has(baseDepID(node.id))) {
|
|
289
297
|
this.ok = false;
|
|
290
298
|
return;
|
|
@@ -294,28 +302,39 @@ export class SecurityArchive extends LRUCache {
|
|
|
294
302
|
this.ok = true;
|
|
295
303
|
}
|
|
296
304
|
/**
|
|
297
|
-
* Starts the security archive by providing
|
|
305
|
+
* Starts the security archive by providing an array of {@link NodeLike} instances,
|
|
298
306
|
* its registry-based nodes are going to be used as valid potential entries.
|
|
299
307
|
*
|
|
300
308
|
* Any entry that is missing from the persisted cached values are going
|
|
301
309
|
* to be requested in a batch-request to the remote socket.dev API.
|
|
302
310
|
*/
|
|
303
|
-
async refresh({
|
|
311
|
+
async refresh({ nodes }) {
|
|
304
312
|
// should start by clearing the current in-memory cache
|
|
305
313
|
this.clear();
|
|
306
314
|
const db = this.#openDatabase();
|
|
315
|
+
// indexes nodes by name and id for quick lookup
|
|
316
|
+
this.#nodesByName.clear();
|
|
317
|
+
this.#nodesByID.clear();
|
|
318
|
+
for (const node of nodes) {
|
|
319
|
+
/* c8 ignore start */
|
|
320
|
+
const setByName = this.#nodesByName.get(node.name ?? '') ?? new Set();
|
|
321
|
+
setByName.add(node);
|
|
322
|
+
this.#nodesByName.set(node.name ?? '', setByName);
|
|
323
|
+
/* c8 ignore stop */
|
|
324
|
+
this.#nodesByID.set(baseDepID(node.id), node);
|
|
325
|
+
}
|
|
307
326
|
try {
|
|
308
|
-
this.#loadItemsFromDatabase(db,
|
|
309
|
-
const queue = this.#queueUpRequiredPackages(
|
|
327
|
+
this.#loadItemsFromDatabase(db, nodes);
|
|
328
|
+
const queue = this.#queueUpRequiredPackages();
|
|
310
329
|
// only reach for the remote API if there are packages queued up
|
|
311
330
|
if (queue.size > 0) {
|
|
312
331
|
// Parse the response data
|
|
313
332
|
const res = await pRetry(() => this.#retrieveRemoteData(queue), { retries: this.#retries });
|
|
314
|
-
const ids = this.#loadFromNDJSON(res
|
|
333
|
+
const ids = this.#loadFromNDJSON(res);
|
|
315
334
|
this.#storeNewItemsToDatabase(db, ids);
|
|
316
335
|
}
|
|
317
336
|
// validates the refresh process was successful
|
|
318
|
-
this.#validateReportData(
|
|
337
|
+
this.#validateReportData();
|
|
319
338
|
}
|
|
320
339
|
finally {
|
|
321
340
|
// TODO: once we move the stale-while-revalidate to a detached process
|
package/dist/esm/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AACnC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpC,OAAO,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AACxD,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC/D,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAC3C,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAUhD,cAAc,YAAY,CAAA;AAE1B,MAAM,iBAAiB,GAAG,4CAA4C,CAAA;AACtE,MAAM,uBAAuB,GAC3B,yDAAyD,CAAA;AAE3D,MAAM,CAAC,MAAM,qBAAqB,GAAG,6BAA6B,CAAA;AA6ClE;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,CACzB,KAAY,EACZ,WAAwB,EACf,EAAE;IACX,MAAM,CAAC,QAAQ,EAAE,YAAY,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;IAElD,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;QAC5B,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,GAAG,GACP,YAAY,KAAK,EAAE,CAAC,CAAC;QACnB,WAAW,CAAC,QAAQ;QACtB,CAAC,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,YAAY,CAAC,CAAA;IAC1C,OAAO,GAAG,KAAK,qBAAqB,CAAA;AACtC,CAAC,CAAA;AAED,+DAA+D;AAC/D,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CACxC,MAAM,CAAC,IAAI,CAAC,QAAQ,EACpB,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAG5C,CAAA;AAED;;GAEG;AACH,MAAM,qBAAqB,GAAG,CAC5B,KAAgB,EAChB,IAAY,EACZ,OAAe,EACO,EAAE;IACxB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACrD,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAA;QACb,CAAC;IACH,CAAC;AACH,CAAC,CAAA;AAED;;;;;;;GAOG;AACH,MAAM,OAAO,eACX,SAAQ,QAAkC;IAG1C,QAAQ,GAAG,IAAI,GAAG,EAAS,CAAA;IAC3B,eAAe,CAA2B;IAC1C,KAAK,CAAQ;IACb,QAAQ,CAAQ;IAEhB;;;OAGG;IACH,EAAE,GAAG,KAAK,CAAA;IAEV;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,KAAK,CAChB,OAA+D;QAE/D,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAA;QAC5C,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QAC9B,OAAO,OAAO,CAAA;IAChB,CAAC;IAED;;OAEG;IACH,MAAM,KAAK,UAAU;QACnB,OAAO,OAAO,CAAA;IAChB,CAAC;IAED;;OAEG;IACH,MAAM,KAAK,UAAU;QACnB,OAAO,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IAC3B,CAAC;IAED,YAAY,UAAkC,EAAE;QAC9C,KAAK,CAAC;YACJ,GAAG,EAAE,eAAe,CAAC,UAAU;YAC/B,GAAG,EAAE,eAAe,CAAC,UAAU;YAC/B,GAAG,OAAO;YACV,UAAU,EAAE,IAAI;SACjB,CAAC,CAAA;QAEF,IAAI,CAAC,KAAK;YACR,OAAO,CAAC,IAAI,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAA;QAC7D,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAA;IACtC,CAAC;IAED;;OAEG;IACH,aAAa;QACX,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACnD,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACvC,EAAE,CAAC,IAAI,CACL,mCAAmC;YACjC,oEAAoE;YACpE,4CAA4C;YAC5C,8CAA8C;YAC9C,oDAAoD;YACpD,2CAA2C;YAC3C,eAAe,CAClB,CAAA;QACD,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAA;QACpC,EAAE,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAA;QACtC,OAAO,EAAE,CAAA;IACX,CAAC;IAED;;OAEG;IACH,sBAAsB,CAAC,EAAgB,EAAE,KAAgB;QACvD,MAAM,MAAM,GAAa,EAAE,CAAA;QAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAA;QACxC,CAAC;QACD,8DAA8D;QAC9D,uDAAuD;QACvD,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CACvB,oCAAoC;YAClC,gDAAgD;YAChD,aAAa;YACb,mBAAmB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CACzC,CAAA;QACD,oCAAoC;QACpC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;QACrB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,GAAG,EAAmB,EAAE,CAAC;YAClD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,KAAK,CAAA;YAChD,MAAM,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;YACpC,IAAI,CAAC;gBACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE;oBACpD,GAAG;oBACH,KAAK;iBACN,CAAC,CAAA;gBACF,yDAAyD;gBACzD,IAAI,KAAK,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC;oBACtB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;gBACvB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,8CAA8C;gBAC9C,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YAC5D,CAAC;QACH,CAAC;QACD,oEAAoE;QACpE,qEAAqE;QACrE,iEAAiE;QACjE,gDAAgD;QAChD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;IACvD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAClB,EAAgB,EAChB,KAAgB;QAEhB,MAAM,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAA;QACtD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAA;YAE9C,oBAAoB;YACpB,IAAI,CAAC,IAAI;gBAAE,SAAQ;YACnB,MAAM,IAAI,GAAG,WAAW,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA;YACnD,YAAY,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;QAC5B,CAAC;QACD,IAAI,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,MAAM,MAAM,CACtB,GAAG,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,EAC5C,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,CAC3B,CAAA;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;YAC5C,IAAI,CAAC,wBAAwB,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;QACxC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,wBAAwB,CACtB,KAAgB,EAChB,WAAwB;QAExB,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAA;QAC/C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAA;QAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACvC,sCAAsC;YACtC,+CAA+C;YAC/C,uCAAuC;YACvC,IACE,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC;gBACxC,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC;gBACvB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,EAChC,CAAC;gBACD,MAAM,IAAI,GAAG,WAAW,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA;gBACnD,KAAK,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YACrB,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,mBAAmB,CACvB,KAAkC;QAElC,4CAA4C;QAC5C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,iBAAiB,EAAE;YACzC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG,uBAAuB,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE;gBAC1F,YAAY,EAAE,4BAA4B,OAAO,EAAE;aACpD;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;aAC9B,CAAC;SACH,CAAC,CAAA;QACF,gEAAgE;QAChE,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACvB,MAAM,IAAI,UAAU,CAAC,aAAa,CAAC,CAAA;QACrC,CAAC;QACD,6DAA6D;QAC7D,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC;YACzD,MAAM,KAAK,CAAC,+BAA+B,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAA;QACjE,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAA;QAC5B,OAAO,GAAG,CAAC,IAAI,EAAE,GAAG,IAAI,CAAA;IAC1B,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,GAAW,EAAE,KAAgB;QAC3C,oEAAoE;QACpE,MAAM,aAAa,GAAG,IAAI,GAAG,EAAS,CAAA;QACtC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAC7B,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;gBAAE,SAAQ;YAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAqB,CAAA;YACvD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;YACxD,MAAM,IAAI,GAAG,GAAG,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;YACnC,MAAM,IAAI,GAAG,qBAAqB,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YAC7D,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;gBACvC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;gBACrC,oDAAoD;gBACpD,MAAM,eAAe,GAAG;oBACtB,IAAI,CAAC,KAAK,CAAC,OAAO;oBAClB,IAAI,CAAC,KAAK,CAAC,WAAW;oBACtB,IAAI,CAAC,KAAK,CAAC,OAAO;oBAClB,IAAI,CAAC,KAAK,CAAC,WAAW;oBACtB,IAAI,CAAC,KAAK,CAAC,aAAa;iBACzB,CAAA;gBACD,MAAM,eAAe,GAAG,MAAM,CAC5B,CACE,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC;oBACtD,eAAe,CAAC,MAAM,CACvB,CAAC,OAAO,CAAC,CAAC,CAAC,CACb,CAAA;gBACD,wCAAwC;gBACxC,MAAM,mBAAmB,GAAG;oBAC1B,GAAG,IAAI,CAAC,KAAK;oBACb,OAAO,EAAE,eAAe;iBACzB,CAAA;gBACD,IAAI,CAAC,GAAG,CACN,YAAY,EACZ,mBAAmB,CAAC;oBAClB,GAAG,IAAI;oBACP,KAAK,EAAE,mBAAmB;iBAC3B,CAAC,CACH,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,sCAAsC;gBACtC,OAAO,CAAC,IAAI,CACV,6CAA6C,KAAK,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,yBAAyB,CACxG,CAAA;YACH,CAAC;QACH,CAAC;QACD,OAAO,aAAa,CAAA;IACtB,CAAC;IAED;;OAEG;IACH,wBAAwB,CACtB,EAAgB,EAChB,MAAkB;QAElB,MAAM,UAAU,GAAmB,EAAE,CAAA;QACrC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9B,IAAI,KAAK,EAAE,KAAK,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;gBAC9B,UAAU,CAAC,IAAI,CAAC;oBACd,KAAK;oBACL,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC;oBAC3B,KAAK,CAAC,KAAK;oBACX,KAAK,CAAC,GAAG;iBACV,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;QACD,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,CACxB,2DAA2D;YACzD,qBAAqB,CACxB,CAAA;QACD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;QACtB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,KAAgB,EAAE,WAAwB;QAC5D,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,IAAI,kBAAkB,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,EAAE,CAAC;gBAC7C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;oBAClC,IAAI,CAAC,EAAE,GAAG,KAAK,CAAA;oBACf,OAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,EAAE,GAAG,IAAI,CAAA;IAChB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CAAC,EACZ,KAAK,EACL,WAAW,GACmB;QAC9B,uDAAuD;QACvD,IAAI,CAAC,KAAK,EAAE,CAAA;QAEZ,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,EAAE,CAAA;QAE/B,IAAI,CAAC;YACH,IAAI,CAAC,sBAAsB,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;YAEtC,MAAM,KAAK,GAAG,IAAI,CAAC,wBAAwB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAA;YAC/D,gEAAgE;YAChE,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACnB,0BAA0B;gBAC1B,MAAM,GAAG,GAAG,MAAM,MAAM,CACtB,GAAG,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,EACrC,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,CAC3B,CAAA;gBACD,MAAM,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;gBAC5C,IAAI,CAAC,wBAAwB,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;YACxC,CAAC;YAED,+CAA+C;YAC/C,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAA;QAC9C,CAAC;gBAAS,CAAC;YACT,sEAAsE;YACtE,qDAAqD;YACrD,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QACtB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAgB;QAC3B,+DAA+D;QAC/D,iEAAiE;QACjE,MAAM,IAAI,CAAC,eAAe,CAAA;QAC1B,sBAAsB;QACtB,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QAC1B,EAAE,CAAC,KAAK,EAAE,CAAA;QACV,0BAA0B;QAC1B,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;QACrB,IAAI,CAAC,eAAe,GAAG,SAAS,CAAA;QAChC,wDAAwD;QACxD,2DAA2D;IAC7D,CAAC;IAED;;OAEG;IACH,MAAM;QACJ,MAAM,GAAG,GAAqC,EAAE,CAAA;QAChD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACvC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAA;QACxB,CAAC;QACD,OAAO,GAAG,CAAA;IACZ,CAAC;CACF","sourcesContent":["import { DatabaseSync } from 'node:sqlite'\nimport { mkdirSync } from 'node:fs'\nimport { dirname } from 'node:path'\nimport { LRUCache } from 'lru-cache'\nimport pRetry, { AbortError } from 'p-retry'\nimport { loadPackageJson } from 'package-json-from-dist'\nimport { asDepID, splitDepID, baseDepID } from '@vltpkg/dep-id'\nimport { error } from '@vltpkg/error-cause'\nimport { XDG } from '@vltpkg/xdg'\nimport { asPackageReportData } from './types.ts'\nimport type { DepID } from '@vltpkg/dep-id'\nimport type { GraphLike, NodeLike } from '@vltpkg/graph'\nimport type { SpecOptions } from '@vltpkg/spec'\nimport type {\n PackageReportData,\n SecurityArchiveLike,\n SecurityArchiveRefreshOptions,\n} from './types.ts'\n\nexport * from './types.ts'\n\nconst SOCKET_API_V0_URL = 'https://api.socket.dev/v0/purl?alerts=true'\nconst SOCKET_PUBLIC_API_TOKEN =\n 'sktsec_t_--RAN5U4ivauy4w37-6aoKyYPDt5ZbaT5JBVMqiwKo_api'\n\nexport const targetSecurityRegisty = 'https://registry.npmjs.org/'\n\nexport type JSONItemResponse = {\n namespace?: `@{string}`\n name: string\n version: string\n score: {\n overall: number\n license: number\n maintenance: number\n quality: number\n supplyChain: number\n vulnerability: number\n }\n}\n\nexport type DBReadEntry = {\n depID: string\n now: number\n report: string\n start: number\n ttl: number\n}\n\nexport type DBWriteEntry = [string, string, number, number]\n\nexport type SecurityArchiveOptions = LRUCache.OptionsBase<\n DepID,\n PackageReportData,\n unknown\n> & {\n /**\n * Security archive does not supports a fetch-on-demand model.\n */\n fetchMethod?: undefined\n /**\n * An optional value for the path in which to store the sqlite db.\n */\n path?: string\n /**\n * Number of retries attempts to reach the remote security API.\n */\n retries?: number\n}\n\n/**\n * Only the public npm registry has information available in the\n * socket.dev API. This function checks if a given depID is pointing\n * to the public npm registry and returns `true` if it does.\n */\nconst usesTargetRegistry = (\n depID: DepID,\n specOptions: SpecOptions,\n): boolean => {\n const [nodeType, nodeRegistry] = splitDepID(depID)\n\n if (nodeType !== 'registry') {\n return false\n }\n\n const reg =\n nodeRegistry === '' ?\n specOptions.registry\n : specOptions.registries?.[nodeRegistry]\n return reg === targetSecurityRegisty\n}\n\n// Loads the version number to be used in the User-Agent header\nexport const { version } = loadPackageJson(\n import.meta.filename,\n process.env.__VLT_INTERNAL_CLI_PACKAGE_JSON,\n) as {\n version: string\n}\n\n/**\n * Retrieves a node from the graph using its name and version.\n */\nconst retrieveNodeFromGraph = (\n graph: GraphLike,\n name: string,\n version: string,\n): NodeLike | undefined => {\n for (const node of graph.nodesByName.get(name) ?? []) {\n if (node.version === version) {\n return node\n }\n }\n}\n\n/**\n * A database of security information for given packages in a graph.\n *\n * Using the SecurityArchive.refresh() method will update the local cache\n * with information from the socket.dev APIs or load from the local storage\n * if available. Information about package security is then available\n * using the SecurityArchive.get() method.\n */\nexport class SecurityArchive\n extends LRUCache<DepID, PackageReportData>\n implements SecurityArchiveLike\n{\n #expired = new Set<DepID>()\n #pUpdateExpired: Promise<void> | undefined\n #path: string\n #retries: number\n\n /**\n * True if the refresh process was successful and report data\n * is available for all public registry packages in the graph.\n */\n ok = false\n\n /**\n * Creates a new security archive instance and starts the refresh process.\n */\n static async start(\n options: SecurityArchiveOptions & SecurityArchiveRefreshOptions,\n ) {\n const archive = new SecurityArchive(options)\n await archive.refresh(options)\n return archive\n }\n\n /**\n * By default, limits to 100K entries in the in-memory archive.\n */\n static get defaultMax() {\n return 100_000\n }\n\n /**\n * By default, entries are cached for 3 hours.\n */\n static get defaultTtl() {\n return 1000 * 60 * 60 * 3\n }\n\n constructor(options: SecurityArchiveOptions = {}) {\n super({\n max: SecurityArchive.defaultMax,\n ttl: SecurityArchive.defaultTtl,\n ...options,\n allowStale: true,\n })\n\n this.#path =\n options.path ?? new XDG('vlt').cache('security-archive.db')\n this.#retries = options.retries ?? 3\n }\n\n /**\n * Opens a connection to the database at the given path.\n */\n #openDatabase(): DatabaseSync {\n mkdirSync(dirname(this.#path), { recursive: true })\n const db = new DatabaseSync(this.#path)\n db.exec(\n 'CREATE TABLE IF NOT EXISTS cache ' +\n '(depID TEXT PRIMARY KEY, report TEXT, ttl INTEGER, start INTEGER) ' +\n // WITHOUT ROWID models a key=value storage,\n // enforcing depID must be unique and not null\n // while using less space and more efficient lookups\n // https://www.sqlite.org/withoutrowid.html\n 'WITHOUT ROWID',\n )\n db.exec('PRAGMA journal_mode = WAL')\n db.exec('PRAGMA synchronous = NORMAL')\n return db\n }\n\n /**\n * Loads all valid items found in the database into the in-memory cache.\n */\n #loadItemsFromDatabase(db: DatabaseSync, graph: GraphLike): void {\n const depIDs: string[] = []\n for (const node of graph.nodes.values()) {\n depIDs.push(`'${baseDepID(node.id)}'`)\n }\n // retrieves from the db packages using their dep-id reference\n // and only include entries that have a valid TTL value\n const dbRead = db.prepare(\n 'SELECT depID, report, start, ttl, ' +\n `(SELECT UNIXEPOCH('subsecond') * 1000) as now ` +\n 'FROM cache ' +\n `WHERE depID IN (${depIDs.join(',')})`,\n )\n // reset the list of expired entries\n this.#expired.clear()\n for (const entry of dbRead.all() as DBReadEntry[]) {\n const { depID, now, report, start, ttl } = entry\n const id = baseDepID(asDepID(depID))\n try {\n this.set(id, asPackageReportData(JSON.parse(report)), {\n ttl,\n start,\n })\n // stale values are queued up as expired for revalidation\n if (start + ttl < now) {\n this.#expired.add(id)\n }\n } catch {\n // prune any invalid entries from the database\n db.prepare('DELETE FROM cache WHERE depID = ?').run(depID)\n }\n }\n // TODO: we need to move this to a detached process in order for the\n // cli commands that make usage of the security-archive, e.g: vlt ls,\n // vlt query to not hang while waiting for stale-while-revalidate\n // request to finish and close the db connection\n this.#pUpdateExpired = this.#updateExpired(db, graph)\n }\n\n /**\n * Updates the database with renewed entries for expired packages.\n */\n async #updateExpired(\n db: DatabaseSync,\n graph: GraphLike,\n ): Promise<void> {\n const expiredQueue = new Set<Record<'purl', string>>()\n for (const depID of this.#expired) {\n const node = graph.nodes.get(baseDepID(depID))\n\n /* c8 ignore next */\n if (!node) continue\n const purl = `pkg:npm/${node.name}@${node.version}`\n expiredQueue.add({ purl })\n }\n if (expiredQueue.size > 0) {\n const res = await pRetry(\n () => this.#retrieveRemoteData(expiredQueue),\n { retries: this.#retries },\n )\n const ids = this.#loadFromNDJSON(res, graph)\n this.#storeNewItemsToDatabase(db, ids)\n }\n }\n\n /**\n * Queues up all required packages that are missing from the archive.\n */\n #queueUpRequiredPackages(\n graph: GraphLike,\n specOptions: SpecOptions,\n ): Set<Record<'purl', string>> {\n const queue = new Set<Record<'purl', string>>()\n const nodes = graph.nodes.values()\n for (const node of nodes) {\n const normalizedId = baseDepID(node.id)\n // only queue up valid public registry\n // references that are missing from the archive\n // also skips any pkg marked as expired\n if (\n usesTargetRegistry(node.id, specOptions) &&\n !this.has(normalizedId) &&\n !this.#expired.has(normalizedId)\n ) {\n const purl = `pkg:npm/${node.name}@${node.version}`\n queue.add({ purl })\n }\n }\n return queue\n }\n\n /**\n * Fetches security information from the socket.dev API.\n * Returns a string of NDJSON data.\n */\n async #retrieveRemoteData(\n queue: Set<Record<'purl', string>>,\n ): Promise<string> {\n // fetch information from the socket.dev API\n const req = await fetch(SOCKET_API_V0_URL, {\n method: 'POST',\n headers: {\n Authorization: `Basic ${Buffer.from(`${SOCKET_PUBLIC_API_TOKEN}:`).toString('base64url')}`,\n 'User-Agent': `@vltpkg/security-archive/${version}`,\n },\n body: JSON.stringify({\n components: Array.from(queue),\n }),\n })\n // on missing valid auth or API, it should abort the retry logic\n if (req.status === 404) {\n throw new AbortError('Missing API')\n }\n // on any other error throws an error that will trigger retry\n if (!req.ok || !(req.status >= 200 && req.status <= 299)) {\n throw error('Failed to fetch security data', { response: req })\n }\n\n const str = await req.text()\n return str.trim() + '\\n'\n }\n\n /**\n * Parses and load NDJSON data into the in-memory cache.\n * Returns a set of dep ids that were successfully loaded.\n */\n #loadFromNDJSON(str: string, graph: GraphLike): Set<DepID> {\n // builds a set of dep ids that were successfully fetched and loaded\n const fetchedDepIDs = new Set<DepID>()\n const json = str.split('}\\n')\n for (const line of json) {\n if (!line.trim()) continue\n const data = JSON.parse(line + '}') as JSONItemResponse\n const scope = data.namespace ? `${data.namespace}/` : ''\n const name = `${scope}${data.name}`\n const node = retrieveNodeFromGraph(graph, name, data.version)\n if (node) {\n const normalizedId = baseDepID(node.id)\n fetchedDepIDs.add(baseDepID(node.id))\n // Calculate average score from all score components\n const scoreComponents = [\n data.score.license,\n data.score.maintenance,\n data.score.quality,\n data.score.supplyChain,\n data.score.vulnerability,\n ]\n const newAverageScore = Number(\n (\n scoreComponents.reduce((sum, score) => sum + score, 0) /\n scoreComponents.length\n ).toFixed(2),\n )\n // Add average score to the score object\n const scoreWithNewAverage = {\n ...data.score,\n overall: newAverageScore,\n }\n this.set(\n normalizedId,\n asPackageReportData({\n ...data,\n score: scoreWithNewAverage,\n }),\n )\n } else {\n // eslint-disable-next-line no-console\n console.warn(\n `security-archive: failed to find node for ${scope}${data.name}@${data.version} found in the response.`,\n )\n }\n }\n return fetchedDepIDs\n }\n\n /**\n * Store new in-memory items to the database.\n */\n #storeNewItemsToDatabase(\n db: DatabaseSync,\n depIDs: Set<DepID>,\n ): void {\n const insertData: DBWriteEntry[] = []\n for (const depID of depIDs) {\n const entry = this.info(depID)\n if (entry?.start && entry.ttl) {\n insertData.push([\n depID,\n JSON.stringify(entry.value),\n entry.start,\n entry.ttl,\n ])\n }\n }\n const dbWrite = db.prepare(\n 'INSERT OR REPLACE INTO cache (depID, report, start, ttl) ' +\n 'VALUES (?, ?, ?, ?)',\n )\n for (const data of insertData) {\n dbWrite.run(...data)\n }\n }\n\n /**\n * Validates that all public-registry packages in the graph\n * have a valid report data in the current in-memory cache.\n */\n #validateReportData(graph: GraphLike, specOptions: SpecOptions) {\n for (const node of graph.nodes.values()) {\n if (usesTargetRegistry(node.id, specOptions)) {\n if (!this.has(baseDepID(node.id))) {\n this.ok = false\n return\n }\n }\n }\n this.ok = true\n }\n\n /**\n * Starts the security archive by providing a {@link GraphLike} instance,\n * its registry-based nodes are going to be used as valid potential entries.\n *\n * Any entry that is missing from the persisted cached values are going\n * to be requested in a batch-request to the remote socket.dev API.\n */\n async refresh({\n graph,\n specOptions,\n }: SecurityArchiveRefreshOptions) {\n // should start by clearing the current in-memory cache\n this.clear()\n\n const db = this.#openDatabase()\n\n try {\n this.#loadItemsFromDatabase(db, graph)\n\n const queue = this.#queueUpRequiredPackages(graph, specOptions)\n // only reach for the remote API if there are packages queued up\n if (queue.size > 0) {\n // Parse the response data\n const res = await pRetry(\n () => this.#retrieveRemoteData(queue),\n { retries: this.#retries },\n )\n const ids = this.#loadFromNDJSON(res, graph)\n this.#storeNewItemsToDatabase(db, ids)\n }\n\n // validates the refresh process was successful\n this.#validateReportData(graph, specOptions)\n } finally {\n // TODO: once we move the stale-while-revalidate to a detached process\n // the this.#close method no longer needs to be async\n void this.#close(db)\n }\n }\n\n /**\n * Closes the database connection and cleans up the internal state.\n */\n async #close(db: DatabaseSync) {\n // the revalidation of stale entries might not yet be completed\n // so in case we have a pending promise for that, we need to wait\n await this.#pUpdateExpired\n // close db connection\n db.exec('PRAGMA optimize')\n db.close()\n // clean up internal state\n this.#expired.clear()\n this.#pUpdateExpired = undefined\n // TODO: at this point we could spawn a deref process to\n // remove entries with expired ttl values and vaccum the db\n }\n\n /**\n * Outputs the current in-memory cache as a JSON object.\n */\n toJSON() {\n const obj: Record<DepID, PackageReportData> = {}\n for (const [key, value] of this.dump()) {\n obj[key] = value.value\n }\n return obj\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AACnC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AACpC,OAAO,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AACxD,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC/D,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAC3C,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAShD,cAAc,YAAY,CAAA;AAE1B,MAAM,iBAAiB,GAAG,4CAA4C,CAAA;AACtE,MAAM,uBAAuB,GAC3B,yDAAyD,CAAA;AAE3D,MAAM,CAAC,MAAM,qBAAqB,GAAG,6BAA6B,CAAA;AA6ClE;;;;GAIG;AACH,MAAM,kBAAkB,GAAG,CAAC,IAAc,EAAW,EAAE;IACrD,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAA;IACrB,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAA;IAChC,MAAM,CAAC,QAAQ,EAAE,YAAY,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;IAElD,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;QAC5B,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,GAAG,GACP,YAAY,KAAK,EAAE,CAAC,CAAC;QACnB,WAAW,CAAC,QAAQ;QACtB,CAAC,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,YAAY,CAAC,CAAA;IAC1C,OAAO,GAAG,KAAK,qBAAqB,CAAA;AACtC,CAAC,CAAA;AAED,+DAA+D;AAC/D,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CACxC,MAAM,CAAC,IAAI,CAAC,QAAQ,EACpB,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAG5C,CAAA;AAED;;;;;;;GAOG;AACH,MAAM,OAAO,eACX,SAAQ,QAAkC;IAG1C,QAAQ,GAAG,IAAI,GAAG,EAAS,CAAA;IAC3B,eAAe,CAA2B;IAC1C,KAAK,CAAQ;IACb,QAAQ,CAAQ;IAChB,YAAY,GAAG,IAAI,GAAG,EAAyB,CAAA;IAC/C,UAAU,GAAG,IAAI,GAAG,EAAmB,CAAA;IAEvC;;;OAGG;IACH,EAAE,GAAG,KAAK,CAAA;IAEV;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,KAAK,CAChB,OAA+D;QAE/D,MAAM,OAAO,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAA;QAC5C,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QAC9B,OAAO,OAAO,CAAA;IAChB,CAAC;IAED;;OAEG;IACH,MAAM,KAAK,UAAU;QACnB,OAAO,OAAO,CAAA;IAChB,CAAC;IAED;;OAEG;IACH,MAAM,KAAK,UAAU;QACnB,OAAO,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IAC3B,CAAC;IAED,YAAY,UAAkC,EAAE;QAC9C,KAAK,CAAC;YACJ,GAAG,EAAE,eAAe,CAAC,UAAU;YAC/B,GAAG,EAAE,eAAe,CAAC,UAAU;YAC/B,GAAG,OAAO;YACV,UAAU,EAAE,IAAI;SACjB,CAAC,CAAA;QAEF,IAAI,CAAC,KAAK;YACR,OAAO,CAAC,IAAI,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAA;QAC7D,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAA;IACtC,CAAC;IAED;;OAEG;IACH,0BAA0B,CACxB,IAAY,EACZ,OAAe;QAEf,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACrD,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;gBAC7B,OAAO,IAAI,CAAA;YACb,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa;QACX,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACnD,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACvC,EAAE,CAAC,IAAI,CACL,mCAAmC;YACjC,oEAAoE;YACpE,4CAA4C;YAC5C,8CAA8C;YAC9C,oDAAoD;YACpD,2CAA2C;YAC3C,eAAe,CAClB,CAAA;QACD,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAA;QACpC,EAAE,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAA;QACtC,OAAO,EAAE,CAAA;IACX,CAAC;IAED;;OAEG;IACH,sBAAsB,CAAC,EAAgB,EAAE,KAAiB;QACxD,MAAM,MAAM,GAAa,EAAE,CAAA;QAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAA;QACxC,CAAC;QACD,8DAA8D;QAC9D,uDAAuD;QACvD,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CACvB,oCAAoC;YAClC,gDAAgD;YAChD,aAAa;YACb,mBAAmB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CACzC,CAAA;QACD,oCAAoC;QACpC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;QACrB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,GAAG,EAAmB,EAAE,CAAC;YAClD,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,KAAK,CAAA;YAChD,MAAM,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;YACpC,IAAI,CAAC;gBACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE;oBACpD,GAAG;oBACH,KAAK;iBACN,CAAC,CAAA;gBACF,yDAAyD;gBACzD,IAAI,KAAK,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC;oBACtB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;gBACvB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,8CAA8C;gBAC9C,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YAC5D,CAAC;QACH,CAAC;QACD,oEAAoE;QACpE,qEAAqE;QACrE,iEAAiE;QACjE,gDAAgD;QAChD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAA;IAChD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,EAAgB;QACnC,MAAM,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAA;QACtD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAA;YAElD,qBAAqB;YACrB,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;gBACnB,yDAAyD;gBACzD,0DAA0D;gBAC1D,sDAAsD;gBACtD,SAAQ;YACV,CAAC;YACD,oBAAoB;YAEpB,MAAM,IAAI,GAAG,WAAW,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA;YACnD,YAAY,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;QAC5B,CAAC;QACD,IAAI,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,MAAM,MAAM,CACtB,GAAG,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,EAC5C,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,CAC3B,CAAA;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAA;YACrC,IAAI,CAAC,wBAAwB,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;QACxC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,wBAAwB;QACtB,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAA;QAC/C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5C,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACvC,sCAAsC;YACtC,+CAA+C;YAC/C,uCAAuC;YACvC,IACE,kBAAkB,CAAC,IAAI,CAAC;gBACxB,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC;gBACvB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,EAChC,CAAC;gBACD,MAAM,IAAI,GAAG,WAAW,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA;gBACnD,KAAK,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAA;YACrB,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,mBAAmB,CACvB,KAAkC;QAElC,4CAA4C;QAC5C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,iBAAiB,EAAE;YACzC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG,uBAAuB,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE;gBAC1F,YAAY,EAAE,4BAA4B,OAAO,EAAE;aACpD;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;aAC9B,CAAC;SACH,CAAC,CAAA;QACF,gEAAgE;QAChE,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACvB,MAAM,IAAI,UAAU,CAAC,aAAa,CAAC,CAAA;QACrC,CAAC;QACD,6DAA6D;QAC7D,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC;YACzD,MAAM,KAAK,CAAC,+BAA+B,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAA;QACjE,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAA;QAC5B,OAAO,GAAG,CAAC,IAAI,EAAE,GAAG,IAAI,CAAA;IAC1B,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,GAAW;QACzB,oEAAoE;QACpE,MAAM,aAAa,GAAG,IAAI,GAAG,EAAS,CAAA;QACtC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAC7B,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;gBAAE,SAAQ;YAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAqB,CAAA;YACvD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;YACxD,MAAM,IAAI,GAAG,GAAG,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;YACnC,MAAM,IAAI,GAAG,IAAI,CAAC,0BAA0B,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YAChE,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;gBACvC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;gBACrC,oDAAoD;gBACpD,MAAM,eAAe,GAAG;oBACtB,IAAI,CAAC,KAAK,CAAC,OAAO;oBAClB,IAAI,CAAC,KAAK,CAAC,WAAW;oBACtB,IAAI,CAAC,KAAK,CAAC,OAAO;oBAClB,IAAI,CAAC,KAAK,CAAC,WAAW;oBACtB,IAAI,CAAC,KAAK,CAAC,aAAa;iBACzB,CAAA;gBACD,MAAM,eAAe,GAAG,MAAM,CAC5B,CACE,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC;oBACtD,eAAe,CAAC,MAAM,CACvB,CAAC,OAAO,CAAC,CAAC,CAAC,CACb,CAAA;gBACD,wCAAwC;gBACxC,MAAM,mBAAmB,GAAG;oBAC1B,GAAG,IAAI,CAAC,KAAK;oBACb,OAAO,EAAE,eAAe;iBACzB,CAAA;gBACD,IAAI,CAAC,GAAG,CACN,YAAY,EACZ,mBAAmB,CAAC;oBAClB,GAAG,IAAI;oBACP,KAAK,EAAE,mBAAmB;iBAC3B,CAAC,CACH,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,sCAAsC;gBACtC,OAAO,CAAC,IAAI,CACV,6CAA6C,KAAK,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,yBAAyB,CACxG,CAAA;YACH,CAAC;QACH,CAAC;QACD,OAAO,aAAa,CAAA;IACtB,CAAC;IAED;;OAEG;IACH,wBAAwB,CACtB,EAAgB,EAChB,MAAkB;QAElB,MAAM,UAAU,GAAmB,EAAE,CAAA;QACrC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9B,IAAI,KAAK,EAAE,KAAK,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;gBAC9B,UAAU,CAAC,IAAI,CAAC;oBACd,KAAK;oBACL,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC;oBAC3B,KAAK,CAAC,KAAK;oBACX,KAAK,CAAC,GAAG;iBACV,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;QACD,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,CACxB,2DAA2D;YACzD,qBAAqB,CACxB,CAAA;QACD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;QACtB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,mBAAmB;QACjB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5C,IAAI,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;oBAClC,IAAI,CAAC,EAAE,GAAG,KAAK,CAAA;oBACf,OAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,EAAE,GAAG,IAAI,CAAA;IAChB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CAAC,EAAE,KAAK,EAAiC;QACpD,uDAAuD;QACvD,IAAI,CAAC,KAAK,EAAE,CAAA;QAEZ,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,EAAE,CAAA;QAE/B,gDAAgD;QAChD,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAA;QACzB,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAA;QACvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,qBAAqB;YACrB,MAAM,SAAS,GACb,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,IAAI,GAAG,EAAE,CAAA;YACrD,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YACnB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE,SAAS,CAAC,CAAA;YACjD,oBAAoB;YACpB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAA;QAC/C,CAAC;QAED,IAAI,CAAC;YACH,IAAI,CAAC,sBAAsB,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;YAEtC,MAAM,KAAK,GAAG,IAAI,CAAC,wBAAwB,EAAE,CAAA;YAC7C,gEAAgE;YAChE,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACnB,0BAA0B;gBAC1B,MAAM,GAAG,GAAG,MAAM,MAAM,CACtB,GAAG,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,EACrC,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,CAC3B,CAAA;gBACD,MAAM,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAA;gBACrC,IAAI,CAAC,wBAAwB,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;YACxC,CAAC;YAED,+CAA+C;YAC/C,IAAI,CAAC,mBAAmB,EAAE,CAAA;QAC5B,CAAC;gBAAS,CAAC;YACT,sEAAsE;YACtE,qDAAqD;YACrD,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QACtB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAgB;QAC3B,+DAA+D;QAC/D,iEAAiE;QACjE,MAAM,IAAI,CAAC,eAAe,CAAA;QAC1B,sBAAsB;QACtB,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QAC1B,EAAE,CAAC,KAAK,EAAE,CAAA;QACV,0BAA0B;QAC1B,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;QACrB,IAAI,CAAC,eAAe,GAAG,SAAS,CAAA;QAChC,wDAAwD;QACxD,2DAA2D;IAC7D,CAAC;IAED;;OAEG;IACH,MAAM;QACJ,MAAM,GAAG,GAAqC,EAAE,CAAA;QAChD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACvC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAA;QACxB,CAAC;QACD,OAAO,GAAG,CAAA;IACZ,CAAC;CACF","sourcesContent":["import { DatabaseSync } from 'node:sqlite'\nimport { mkdirSync } from 'node:fs'\nimport { dirname } from 'node:path'\nimport { LRUCache } from 'lru-cache'\nimport pRetry, { AbortError } from 'p-retry'\nimport { loadPackageJson } from 'package-json-from-dist'\nimport { asDepID, splitDepID, baseDepID } from '@vltpkg/dep-id'\nimport { error } from '@vltpkg/error-cause'\nimport { XDG } from '@vltpkg/xdg'\nimport { asPackageReportData } from './types.ts'\nimport type { DepID } from '@vltpkg/dep-id'\nimport type { NodeLike } from '@vltpkg/types'\nimport type {\n PackageReportData,\n SecurityArchiveLike,\n SecurityArchiveRefreshOptions,\n} from './types.ts'\n\nexport * from './types.ts'\n\nconst SOCKET_API_V0_URL = 'https://api.socket.dev/v0/purl?alerts=true'\nconst SOCKET_PUBLIC_API_TOKEN =\n 'sktsec_t_--RAN5U4ivauy4w37-6aoKyYPDt5ZbaT5JBVMqiwKo_api'\n\nexport const targetSecurityRegisty = 'https://registry.npmjs.org/'\n\nexport type JSONItemResponse = {\n namespace?: `@{string}`\n name: string\n version: string\n score: {\n overall: number\n license: number\n maintenance: number\n quality: number\n supplyChain: number\n vulnerability: number\n }\n}\n\nexport type DBReadEntry = {\n depID: string\n now: number\n report: string\n start: number\n ttl: number\n}\n\nexport type DBWriteEntry = [string, string, number, number]\n\nexport type SecurityArchiveOptions = LRUCache.OptionsBase<\n DepID,\n PackageReportData,\n unknown\n> & {\n /**\n * Security archive does not supports a fetch-on-demand model.\n */\n fetchMethod?: undefined\n /**\n * An optional value for the path in which to store the sqlite db.\n */\n path?: string\n /**\n * Number of retries attempts to reach the remote security API.\n */\n retries?: number\n}\n\n/**\n * Only the public npm registry has information available in the\n * socket.dev API. This function checks if a given depID is pointing\n * to the public npm registry and returns `true` if it does.\n */\nconst usesTargetRegistry = (node: NodeLike): boolean => {\n const depID = node.id\n const specOptions = node.options\n const [nodeType, nodeRegistry] = splitDepID(depID)\n\n if (nodeType !== 'registry') {\n return false\n }\n\n const reg =\n nodeRegistry === '' ?\n specOptions.registry\n : specOptions.registries?.[nodeRegistry]\n return reg === targetSecurityRegisty\n}\n\n// Loads the version number to be used in the User-Agent header\nexport const { version } = loadPackageJson(\n import.meta.filename,\n process.env.__VLT_INTERNAL_CLI_PACKAGE_JSON,\n) as {\n version: string\n}\n\n/**\n * A database of security information for given packages from a list of nodes.\n *\n * Using the SecurityArchive.refresh() method will update the local cache\n * with information from the socket.dev APIs or load from the local storage\n * if available. Information about package security is then available\n * using the SecurityArchive.get() method.\n */\nexport class SecurityArchive\n extends LRUCache<DepID, PackageReportData>\n implements SecurityArchiveLike\n{\n #expired = new Set<DepID>()\n #pUpdateExpired: Promise<void> | undefined\n #path: string\n #retries: number\n #nodesByName = new Map<string, Set<NodeLike>>()\n #nodesByID = new Map<DepID, NodeLike>()\n\n /**\n * True if the refresh process was successful and report data is available\n * for all public registry packages from the initial list of nodes.\n */\n ok = false\n\n /**\n * Creates a new security archive instance and starts the refresh process.\n */\n static async start(\n options: SecurityArchiveOptions & SecurityArchiveRefreshOptions,\n ) {\n const archive = new SecurityArchive(options)\n await archive.refresh(options)\n return archive\n }\n\n /**\n * By default, limits to 100K entries in the in-memory archive.\n */\n static get defaultMax() {\n return 100_000\n }\n\n /**\n * By default, entries are cached for 3 hours.\n */\n static get defaultTtl() {\n return 1000 * 60 * 60 * 3\n }\n\n constructor(options: SecurityArchiveOptions = {}) {\n super({\n max: SecurityArchive.defaultMax,\n ttl: SecurityArchive.defaultTtl,\n ...options,\n allowStale: true,\n })\n\n this.#path =\n options.path ?? new XDG('vlt').cache('security-archive.db')\n this.#retries = options.retries ?? 3\n }\n\n /**\n * Retrieves a node using its name and version.\n */\n #retrieveNodeByNameVersion(\n name: string,\n version: string,\n ): NodeLike | undefined {\n for (const node of this.#nodesByName.get(name) ?? []) {\n if (node.version === version) {\n return node\n }\n }\n }\n\n /**\n * Opens a connection to the database at the given path.\n */\n #openDatabase(): DatabaseSync {\n mkdirSync(dirname(this.#path), { recursive: true })\n const db = new DatabaseSync(this.#path)\n db.exec(\n 'CREATE TABLE IF NOT EXISTS cache ' +\n '(depID TEXT PRIMARY KEY, report TEXT, ttl INTEGER, start INTEGER) ' +\n // WITHOUT ROWID models a key=value storage,\n // enforcing depID must be unique and not null\n // while using less space and more efficient lookups\n // https://www.sqlite.org/withoutrowid.html\n 'WITHOUT ROWID',\n )\n db.exec('PRAGMA journal_mode = WAL')\n db.exec('PRAGMA synchronous = NORMAL')\n return db\n }\n\n /**\n * Loads all valid items found in the database into the in-memory cache.\n */\n #loadItemsFromDatabase(db: DatabaseSync, nodes: NodeLike[]): void {\n const depIDs: string[] = []\n for (const node of nodes) {\n depIDs.push(`'${baseDepID(node.id)}'`)\n }\n // retrieves from the db packages using their dep-id reference\n // and only include entries that have a valid TTL value\n const dbRead = db.prepare(\n 'SELECT depID, report, start, ttl, ' +\n `(SELECT UNIXEPOCH('subsecond') * 1000) as now ` +\n 'FROM cache ' +\n `WHERE depID IN (${depIDs.join(',')})`,\n )\n // reset the list of expired entries\n this.#expired.clear()\n for (const entry of dbRead.all() as DBReadEntry[]) {\n const { depID, now, report, start, ttl } = entry\n const id = baseDepID(asDepID(depID))\n try {\n this.set(id, asPackageReportData(JSON.parse(report)), {\n ttl,\n start,\n })\n // stale values are queued up as expired for revalidation\n if (start + ttl < now) {\n this.#expired.add(id)\n }\n } catch {\n // prune any invalid entries from the database\n db.prepare('DELETE FROM cache WHERE depID = ?').run(depID)\n }\n }\n // TODO: we need to move this to a detached process in order for the\n // cli commands that make usage of the security-archive, e.g: vlt ls,\n // vlt query to not hang while waiting for stale-while-revalidate\n // request to finish and close the db connection\n this.#pUpdateExpired = this.#updateExpired(db)\n }\n\n /**\n * Updates the database with renewed entries for expired packages.\n */\n async #updateExpired(db: DatabaseSync): Promise<void> {\n const expiredQueue = new Set<Record<'purl', string>>()\n for (const depID of this.#expired) {\n const node = this.#nodesByID.get(baseDepID(depID))\n\n /* c8 ignore start */\n if (!node?.version) {\n // skip any missing node or nodes without a version, this\n // should not happen, but some optional dependencies might\n // end up in this state when reading from the lockfile\n continue\n }\n /* c8 ignore stop */\n\n const purl = `pkg:npm/${node.name}@${node.version}`\n expiredQueue.add({ purl })\n }\n if (expiredQueue.size > 0) {\n const res = await pRetry(\n () => this.#retrieveRemoteData(expiredQueue),\n { retries: this.#retries },\n )\n const ids = this.#loadFromNDJSON(res)\n this.#storeNewItemsToDatabase(db, ids)\n }\n }\n\n /**\n * Queues up all required packages that are missing from the archive.\n */\n #queueUpRequiredPackages(): Set<Record<'purl', string>> {\n const queue = new Set<Record<'purl', string>>()\n for (const node of this.#nodesByID.values()) {\n const normalizedId = baseDepID(node.id)\n // only queue up valid public registry\n // references that are missing from the archive\n // also skips any pkg marked as expired\n if (\n usesTargetRegistry(node) &&\n !this.has(normalizedId) &&\n !this.#expired.has(normalizedId)\n ) {\n const purl = `pkg:npm/${node.name}@${node.version}`\n queue.add({ purl })\n }\n }\n return queue\n }\n\n /**\n * Fetches security information from the socket.dev API.\n * Returns a string of NDJSON data.\n */\n async #retrieveRemoteData(\n queue: Set<Record<'purl', string>>,\n ): Promise<string> {\n // fetch information from the socket.dev API\n const req = await fetch(SOCKET_API_V0_URL, {\n method: 'POST',\n headers: {\n Authorization: `Basic ${Buffer.from(`${SOCKET_PUBLIC_API_TOKEN}:`).toString('base64url')}`,\n 'User-Agent': `@vltpkg/security-archive/${version}`,\n },\n body: JSON.stringify({\n components: Array.from(queue),\n }),\n })\n // on missing valid auth or API, it should abort the retry logic\n if (req.status === 404) {\n throw new AbortError('Missing API')\n }\n // on any other error throws an error that will trigger retry\n if (!req.ok || !(req.status >= 200 && req.status <= 299)) {\n throw error('Failed to fetch security data', { response: req })\n }\n\n const str = await req.text()\n return str.trim() + '\\n'\n }\n\n /**\n * Parses and load NDJSON data into the in-memory cache.\n * Returns a set of dep ids that were successfully loaded.\n */\n #loadFromNDJSON(str: string): Set<DepID> {\n // builds a set of dep ids that were successfully fetched and loaded\n const fetchedDepIDs = new Set<DepID>()\n const json = str.split('}\\n')\n for (const line of json) {\n if (!line.trim()) continue\n const data = JSON.parse(line + '}') as JSONItemResponse\n const scope = data.namespace ? `${data.namespace}/` : ''\n const name = `${scope}${data.name}`\n const node = this.#retrieveNodeByNameVersion(name, data.version)\n if (node) {\n const normalizedId = baseDepID(node.id)\n fetchedDepIDs.add(baseDepID(node.id))\n // Calculate average score from all score components\n const scoreComponents = [\n data.score.license,\n data.score.maintenance,\n data.score.quality,\n data.score.supplyChain,\n data.score.vulnerability,\n ]\n const newAverageScore = Number(\n (\n scoreComponents.reduce((sum, score) => sum + score, 0) /\n scoreComponents.length\n ).toFixed(2),\n )\n // Add average score to the score object\n const scoreWithNewAverage = {\n ...data.score,\n overall: newAverageScore,\n }\n this.set(\n normalizedId,\n asPackageReportData({\n ...data,\n score: scoreWithNewAverage,\n }),\n )\n } else {\n // eslint-disable-next-line no-console\n console.warn(\n `security-archive: failed to find node for ${scope}${data.name}@${data.version} found in the response.`,\n )\n }\n }\n return fetchedDepIDs\n }\n\n /**\n * Store new in-memory items to the database.\n */\n #storeNewItemsToDatabase(\n db: DatabaseSync,\n depIDs: Set<DepID>,\n ): void {\n const insertData: DBWriteEntry[] = []\n for (const depID of depIDs) {\n const entry = this.info(depID)\n if (entry?.start && entry.ttl) {\n insertData.push([\n depID,\n JSON.stringify(entry.value),\n entry.start,\n entry.ttl,\n ])\n }\n }\n const dbWrite = db.prepare(\n 'INSERT OR REPLACE INTO cache (depID, report, start, ttl) ' +\n 'VALUES (?, ?, ?, ?)',\n )\n for (const data of insertData) {\n dbWrite.run(...data)\n }\n }\n\n /**\n * Validates that all public-registry packages in the nodes\n * have a valid report data in the current in-memory cache.\n */\n #validateReportData() {\n for (const node of this.#nodesByID.values()) {\n if (usesTargetRegistry(node)) {\n if (!this.has(baseDepID(node.id))) {\n this.ok = false\n return\n }\n }\n }\n this.ok = true\n }\n\n /**\n * Starts the security archive by providing an array of {@link NodeLike} instances,\n * its registry-based nodes are going to be used as valid potential entries.\n *\n * Any entry that is missing from the persisted cached values are going\n * to be requested in a batch-request to the remote socket.dev API.\n */\n async refresh({ nodes }: SecurityArchiveRefreshOptions) {\n // should start by clearing the current in-memory cache\n this.clear()\n\n const db = this.#openDatabase()\n\n // indexes nodes by name and id for quick lookup\n this.#nodesByName.clear()\n this.#nodesByID.clear()\n for (const node of nodes) {\n /* c8 ignore start */\n const setByName =\n this.#nodesByName.get(node.name ?? '') ?? new Set()\n setByName.add(node)\n this.#nodesByName.set(node.name ?? '', setByName)\n /* c8 ignore stop */\n this.#nodesByID.set(baseDepID(node.id), node)\n }\n\n try {\n this.#loadItemsFromDatabase(db, nodes)\n\n const queue = this.#queueUpRequiredPackages()\n // only reach for the remote API if there are packages queued up\n if (queue.size > 0) {\n // Parse the response data\n const res = await pRetry(\n () => this.#retrieveRemoteData(queue),\n { retries: this.#retries },\n )\n const ids = this.#loadFromNDJSON(res)\n this.#storeNewItemsToDatabase(db, ids)\n }\n\n // validates the refresh process was successful\n this.#validateReportData()\n } finally {\n // TODO: once we move the stale-while-revalidate to a detached process\n // the this.#close method no longer needs to be async\n void this.#close(db)\n }\n }\n\n /**\n * Closes the database connection and cleans up the internal state.\n */\n async #close(db: DatabaseSync) {\n // the revalidation of stale entries might not yet be completed\n // so in case we have a pending promise for that, we need to wait\n await this.#pUpdateExpired\n // close db connection\n db.exec('PRAGMA optimize')\n db.close()\n // clean up internal state\n this.#expired.clear()\n this.#pUpdateExpired = undefined\n // TODO: at this point we could spawn a deref process to\n // remove entries with expired ttl values and vaccum the db\n }\n\n /**\n * Outputs the current in-memory cache as a JSON object.\n */\n toJSON() {\n const obj: Record<DepID, PackageReportData> = {}\n for (const [key, value] of this.dump()) {\n obj[key] = value.value\n }\n return obj\n }\n}\n"]}
|
package/dist/esm/types.d.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { DepID } from '@vltpkg/dep-id';
|
|
2
|
-
import type {
|
|
3
|
-
import type { SpecOptions } from '@vltpkg/spec';
|
|
2
|
+
import type { NodeLike } from '@vltpkg/types';
|
|
4
3
|
/**
|
|
5
4
|
* Parameter options for initializing a security archive.
|
|
6
5
|
*/
|
|
@@ -9,11 +8,7 @@ export type SecurityArchiveRefreshOptions = {
|
|
|
9
8
|
* A @link{GraphLike} instance to find what packages the
|
|
10
9
|
* security archive should have.
|
|
11
10
|
*/
|
|
12
|
-
|
|
13
|
-
/**
|
|
14
|
-
* A @link{SpecOptions} instance to use for resolving dependencies.
|
|
15
|
-
*/
|
|
16
|
-
specOptions: SpecOptions;
|
|
11
|
+
nodes: NodeLike[];
|
|
17
12
|
};
|
|
18
13
|
/**
|
|
19
14
|
* An interface for interacting with a security archive.
|
package/dist/esm/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AAC3C,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAA;AAC3C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAA;AAE7C;;GAEG;AACH,MAAM,MAAM,6BAA6B,GAAG;IAC1C;;;OAGG;IACH,KAAK,EAAE,QAAQ,EAAE,CAAA;CAClB,CAAA;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,GAAG,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,iBAAiB,GAAG,SAAS,CAAA;IACpD,GAAG,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,iBAAiB,KAAK,IAAI,CAAA;IACpD,MAAM,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAA;IAC9B,GAAG,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,OAAO,CAAA;IAC9B,KAAK,EAAE,MAAM,IAAI,CAAA;IACjB,EAAE,CAAC,EAAE,OAAO,CAAA;CACb;AAED,eAAO,MAAM,qBAAqB,MAC7B,OAAO,KACT,CAAC,IAAI,mBAOM,CAAA;AAEd,eAAO,MAAM,qBAAqB,MAC7B,OAAO,KACT,mBAKF,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,OAAO,MAAM,EAAE,CAAA;IACvB,IAAI,CAAC,EAAE;QAAE,EAAE,EAAE,OAAO,MAAM,EAAE,CAAA;KAAE,EAAE,CAAA;CACjC,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAA;IAChD,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,iBAAiB,CAAA;CAC1B,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;CACtB,CAAA;AAED;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,KAAK,CAAA;IACX,SAAS,CAAC,EAAE,IAAI,MAAM,EAAE,CAAA;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,YAAY,EAAE,CAAA;IACtB,KAAK,EAAE,YAAY,CAAA;CACpB,CAAA;AAED,eAAO,MAAM,mBAAmB,MAC3B,OAAO,KACT,CAAC,IAAI,iBASU,CAAA;AAElB,eAAO,MAAM,mBAAmB,MAC3B,OAAO,KACT,iBAKF,CAAA"}
|
package/dist/esm/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAA;AA2B3C,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,CAAU,EACgB,EAAE,CAC5B,OAAO,CAAC,KAAK,QAAQ;IACrB,CAAC,IAAI,IAAI;IACT,KAAK,IAAI,CAAC;IACV,KAAK,IAAI,CAAC;IACV,QAAQ,IAAI,CAAC;IACb,KAAK,IAAI,CAAC;IACV,OAAO,IAAI,CAAC,CAAA;AAEd,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,CAAU,EACW,EAAE;IACvB,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9B,MAAM,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;IAC5D,CAAC;IACD,OAAO,CAAC,CAAA;AACV,CAAC,CAAA;AAoED,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,CAAU,EACc,EAAE,CAC1B,OAAO,CAAC,KAAK,QAAQ;IACrB,CAAC,IAAI,IAAI;IACT,IAAI,IAAI,CAAC;IACT,MAAM,IAAI,CAAC;IACX,MAAM,IAAI,CAAC;IACX,SAAS,IAAI,CAAC;IACd,QAAQ,IAAI,CAAC;IACb,OAAO,IAAI,CAAC;IACZ,CAAC,CAAC,IAAI,KAAK,KAAK,CAAA;AAElB,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,CAAU,EACS,EAAE;IACrB,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5B,MAAM,KAAK,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;IAC1D,CAAC;IACD,OAAO,CAAC,CAAA;AACV,CAAC,CAAA","sourcesContent":["import { error } from '@vltpkg/error-cause'\nimport type { DepID } from '@vltpkg/dep-id'\nimport type { NodeLike } from '@vltpkg/types'\n\n/**\n * Parameter options for initializing a security archive.\n */\nexport type SecurityArchiveRefreshOptions = {\n /**\n * A @link{GraphLike} instance to find what packages the\n * security archive should have.\n */\n nodes: NodeLike[]\n}\n\n/**\n * An interface for interacting with a security archive.\n */\nexport interface SecurityArchiveLike {\n get: (depId: DepID) => PackageReportData | undefined\n set: (depId: DepID, data: PackageReportData) => void\n delete: (depId: DepID) => void\n has: (depId: DepID) => boolean\n clear: () => void\n ok?: boolean\n}\n\nexport const isSecurityArchiveLike = (\n o: unknown,\n): o is SecurityArchiveLike =>\n typeof o === 'object' &&\n o != null &&\n 'get' in o &&\n 'set' in o &&\n 'delete' in o &&\n 'has' in o &&\n 'clear' in o\n\nexport const asSecurityArchiveLike = (\n o: unknown,\n): SecurityArchiveLike => {\n if (!isSecurityArchiveLike(o)) {\n throw error('Invalid security archive like', { found: o })\n }\n return o\n}\n\n/**\n * Package alert extra information.\n */\nexport type PackageAlertProps = {\n lastPublish: string\n cveId?: `CVE-${string}`\n cwes?: { id: `CWE-${string}` }[]\n}\n\n/**\n * A known alert for a given package.\n */\nexport type PackageAlert = {\n key: string\n type: string\n severity: 'low' | 'medium' | 'high' | 'critical'\n category: string\n props?: PackageAlertProps\n}\n\n/**\n * The scores for a given package\n */\nexport type PackageScore = {\n /**\n * The average of all score factors. (0-1)\n */\n overall: number\n /**\n * Score factors relating to package licensing (0-1)\n */\n license: number\n /**\n * Score factors relating to package maintenance (0-1)\n */\n maintenance: number\n /**\n * Score factors relating to code quality (0-1)\n */\n quality: number\n /**\n * Score factors relating to supply chain security (0-1)\n */\n supplyChain: number\n /**\n * Score factors relating to package vulnerabilities (0-1)\n */\n vulnerability: number\n}\n\n/**\n * The report data for a given package.\n */\nexport type PackageReportData = {\n id: string\n author: string[]\n size: number\n type: 'npm'\n namespace?: `@${string}`\n name: string\n version: string\n license: string\n alerts: PackageAlert[]\n score: PackageScore\n}\n\nexport const isPackageReportData = (\n o: unknown,\n): o is PackageReportData =>\n typeof o === 'object' &&\n o != null &&\n 'id' in o &&\n 'type' in o &&\n 'name' in o &&\n 'version' in o &&\n 'alerts' in o &&\n 'score' in o &&\n o.type === 'npm'\n\nexport const asPackageReportData = (\n o: unknown,\n): PackageReportData => {\n if (!isPackageReportData(o)) {\n throw error('Invalid package report data', { found: o })\n }\n return o\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vltpkg/security-archive",
|
|
3
3
|
"description": "A database of security information of packages",
|
|
4
|
-
"version": "0.0.0-
|
|
4
|
+
"version": "0.0.0-30",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
7
7
|
"url": "git+https://github.com/vltpkg/vltpkg.git",
|
|
@@ -24,12 +24,12 @@
|
|
|
24
24
|
"lru-cache": "^11.1.0",
|
|
25
25
|
"p-retry": "^7.0.0",
|
|
26
26
|
"package-json-from-dist": "^1.0.1",
|
|
27
|
-
"@vltpkg/
|
|
28
|
-
"@vltpkg/
|
|
29
|
-
"@vltpkg/graph": "0.0.0-
|
|
30
|
-
"@vltpkg/types": "0.0.0-
|
|
31
|
-
"@vltpkg/
|
|
32
|
-
"@vltpkg/
|
|
27
|
+
"@vltpkg/dep-id": "0.0.0-30",
|
|
28
|
+
"@vltpkg/error-cause": "0.0.0-30",
|
|
29
|
+
"@vltpkg/graph": "0.0.0-30",
|
|
30
|
+
"@vltpkg/types": "0.0.0-30",
|
|
31
|
+
"@vltpkg/spec": "0.0.0-30",
|
|
32
|
+
"@vltpkg/xdg": "0.0.0-30"
|
|
33
33
|
},
|
|
34
34
|
"devDependencies": {
|
|
35
35
|
"@eslint/js": "^9.34.0",
|