@ecosyste-ms/critical 1.1.20260822 → 1.3.20260824

package/README.md

@@ -12,14 +12,27 @@ npm install @ecosyste-ms/critical
 
 The package includes a pre-built `critical-packages.db` file:
 
-```javascript
+```typescript
 import { databasePath } from '@ecosyste-ms/critical'
-import Database from 'better-sqlite3'
+import { DatabaseSync } from 'node:sqlite'
 
-const db = new Database(databasePath)
+const db = new DatabaseSync(databasePath)
 const pkg = db.prepare('SELECT * FROM packages WHERE name = ?').get('lodash')
 ```
 
+Written in TypeScript and published with type declarations; the row shapes the build
+side works with (`ApiPackage`, `ApiAdvisory`, `ApiRepoMetadata`, `BuildInfo`) are
+exported as types.
+
+Since 1.3.0 the package has one runtime dependency, [`@ecosyste-ms/ecosystems-ts`][sdk],
+the typed client for the ecosyste.ms API. The build talks to the API through it rather
+than through a hand-rolled fetch layer, so retries, rate limiting and registry routing
+live in one place across the ecosyste.ms packages. The version is pinned exactly, since
+the daily job publishes unattended. Reading the bundled database still touches nothing
+but `node:sqlite`.
+
+[sdk]: https://github.com/ecosyste-ms/ecosystems-ts
+
 See [ecosyste-ms/mcp](https://github.com/ecosyste-ms/mcp) for a full example.
 
 ## Download
@@ -52,6 +65,36 @@ npx @ecosyste-ms/critical -o my-db.db        # custom output path
 npx @ecosyste-ms/critical --stats            # show database statistics
 ```
 
+Set `ECOSYSTEMS_MAILTO` to a contact address before a full build. The ecosyste.ms API
+allows anonymous callers roughly 5,000 requests an hour and a full build makes around
+9,700; a build that identifies itself gets roughly 15,000. Without it most of the version
+fetches are rate-limited away, and `build()` throws rather than publish a database that
+lost more than 5% of its versions.
+
+```bash
+ECOSYSTEMS_MAILTO=you@example.com npx @ecosyste-ms/critical
+```
+
+## Development
+
+```bash
+npm ci
+npm run build          # tsc: lib/*.ts -> dist/
+npm test               # vitest, against lib/ sources -- no build needed
+npm run test:packaged  # packs the tarball and tests what it ships
+npm run lint           # biome check + tsc --noEmit
+npm run format         # biome check --write
+npm run build:db       # build critical-packages.db from the API (what the daily job runs)
+```
+
+The default suite runs against `lib/` and needs no build step. `test:packaged` is
+separate because it builds, runs `npm pack`, and exercises the extracted tarball's
+entrypoint, export surface and `bin` — the checks that catch a broken `exports` map
+rather than broken logic.
+
+The published package is plain JavaScript, so consumers stay on the Node >= 22.5.0 floor
+that `node:sqlite` sets.
+
 ## Schema
 
 ### packages

package/dist/bin.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=bin.d.ts.map

package/dist/bin.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.d.ts","sourceRoot":"","sources":["../lib/bin.ts"],"names":[],"mappings":""}

package/dist/bin.js

@@ -0,0 +1,91 @@
+#!/usr/bin/env node
+import { existsSync } from "node:fs";
+import { DatabaseSync } from "node:sqlite";
+import { build, databasePath } from "./index.js";
+const args = process.argv.slice(2);
+if (args.includes("--help") || args.includes("-h")) {
+    console.log(`Usage: critical [options]
+
+Options:
+  --output, -o <path>   Output database path (default: critical-packages.db)
+  --skip-versions       Skip fetching version data (faster)
+  --stats               Show database statistics
+  --help, -h            Show this help message
+
+Environment:
+  ECOSYSTEMS_MAILTO     Contact address sent to the API. Without it the build runs on
+                        the anonymous rate-limit tier (~5,000 requests/hour against
+                        ~9,700 needed) and aborts once too many version fetches fail.
+
+Examples:
+  npx @ecosyste-ms/critical
+  npx @ecosyste-ms/critical --skip-versions
+  npx @ecosyste-ms/critical -o my-database.db
+  npx @ecosyste-ms/critical --stats`);
+    process.exit(0);
+}
+function getArg(flags) {
+    for (const flag of flags) {
+        const idx = args.indexOf(flag);
+        if (idx !== -1 && args[idx + 1]) {
+            return args[idx + 1] ?? null;
+        }
+    }
+    return null;
+}
+const dbPath = getArg(["--output", "-o"]) || "critical-packages.db";
+if (args.includes("--stats")) {
+    const path = existsSync(dbPath) ? dbPath : databasePath;
+    if (!existsSync(path)) {
+        console.error(`Database not found: ${path}`);
+        process.exit(1);
+    }
+    const db = new DatabaseSync(path, { readOnly: true });
+    const info = db.prepare("SELECT * FROM build_info WHERE id = 1").get();
+    const count = (table) => Number(db.prepare(`SELECT COUNT(*) as count FROM ${table}`).get()?.count ?? 0);
+    const packageCount = count("packages");
+    const versionCount = count("versions");
+    const advisoryCount = count("advisories");
+    const ecosystems = db
+        .prepare(`
+    SELECT ecosystem, COUNT(*) as count
+    FROM packages
+    GROUP BY ecosystem
+    ORDER BY count DESC
+  `)
+        .all();
+    const severities = db
+        .prepare(`
+    SELECT severity, COUNT(*) as count
+    FROM advisories
+    GROUP BY severity
+    ORDER BY count DESC
+  `)
+        .all();
+    console.log(`Database: ${path}`);
+    console.log(`Built: ${info?.built_at || "unknown"}`);
+    console.log(`\nPackages: ${packageCount}`);
+    console.log(`Versions: ${versionCount}`);
+    console.log(`Advisories: ${advisoryCount}`);
+    console.log(`\nBy ecosystem:`);
+    for (const e of ecosystems) {
+        console.log(`  ${e.ecosystem}: ${e.count}`);
+    }
+    console.log(`\nBy severity:`);
+    for (const s of severities) {
+        console.log(`  ${s.severity || "unknown"}: ${s.count}`);
+    }
+    db.close();
+    process.exit(0);
+}
+const skipVersions = args.includes("--skip-versions");
+build({
+    dbPath,
+    fetchVersionsData: !skipVersions,
+})
+    .then(() => process.exit(0))
+    .catch((err) => {
+    console.error(err);
+    process.exit(1);
+});
+//# sourceMappingURL=bin.js.map

package/dist/bin.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bin.js","sourceRoot":"","sources":["../lib/bin.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAkB,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEjE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEnC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC;;;;;;;;;;;;;;;;;oCAiBsB,CAAC,CAAC;IACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,MAAM,CAAC,KAAe;IAC7B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,IAAI,sBAAsB,CAAC;AAEpE,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC;IACxD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,KAAK,CAAC,uBAAuB,IAAI,EAAE,CAAC,CAAC;QAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC,GAAG,EAEvD,CAAC;IACd,MAAM,KAAK,GAAG,CAAC,KAAa,EAAE,EAAE,CAC9B,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,iCAAiC,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;IACjF,MAAM,YAAY,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;IACvC,MAAM,YAAY,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;IACvC,MAAM,aAAa,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,EAAE;SAClB,OAAO,CAAC;;;;;GAKV,CAAC;SACC,GAAG,EAAuD,CAAC;IAC9D,MAAM,UAAU,GAAG,EAAE;SAClB,OAAO,CAAC;;;;;GAKV,CAAC;SACC,GAAG,EAA6D,CAAC;IAEpE,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,QAAQ,IAAI,SAAS,EAAE,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,eAAe,YAAY,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,aAAa,YAAY,EAAE,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,eAAe,aAAa,EAAE,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ,IAAI,SAAS,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IAC1D,CAAC;IACD,EAAE,CAAC,KAAK,EAAE,CAAC;IACX,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;AAEtD,KAAK,CAAC;IACJ,MAAM;IACN,iBAAiB,EAAE,CAAC,YAAY;CACjC,CAAC;KACC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;KAC3B,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,96 @@
+import { DatabaseSync } from "node:sqlite";
+import { type EcosystemsClient, type PackageWithRegistry } from "@ecosyste-ms/ecosystems-ts";
+declare const databasePath: string;
+/**
+ * Every field optional and nullable.
+ *
+ * The OpenAPI spec marks most package fields required and non-null; the API omits them
+ * and sends explicit nulls. Modelling what actually arrives is what keeps the runtime
+ * guards below honest instead of looking redundant.
+ */
+type Loose<T> = {
+    [K in keyof T]?: T[K] | null;
+};
+/**
+ * Repository metadata as `GET /packages/critical` serves it -- every field optional.
+ *
+ * The API sends `full_name` ("owner/repo") but never `name`, and nests the host here
+ * rather than on the package. Both were previously read from the wrong place, which left
+ * `repo_metadata.repo_name` and `repo_metadata.host` NULL in every published build.
+ */
+export interface ApiRepoMetadata {
+    owner?: string | null;
+    name?: string | null;
+    full_name?: string | null;
+    language?: string | null;
+    stargazers_count?: number | null;
+    forks_count?: number | null;
+    open_issues_count?: number | null;
+    archived?: boolean | null;
+    fork?: boolean | null;
+    host?: ApiHost | null;
+}
+/** The registry host, as nested inside `repo_metadata`. */
+export interface ApiHost {
+    name?: string | null;
+}
+/**
+ * An advisory as the API nests it in a package.
+ *
+ * Derived from the package schema rather than the SDK's top-level `Advisory`: the two
+ * specs describe the same objects differently -- `advisories.ecosyste.ms` types the
+ * optional fields `string | undefined`, while the copy embedded in a package uses
+ * `string | null`, which is what the API actually sends.
+ */
+type PackageAdvisory = NonNullable<PackageWithRegistry["advisories"]>[number];
+export type ApiAdvisory = Loose<PackageAdvisory>;
+/**
+ * A package from `GET /packages/critical`, as the SDK types it.
+ *
+ * `repo_metadata` is overridden: the OpenAPI spec types it `Record<string, never>` -- a
+ * known-opaque object -- while this package reads nine fields out of it. Everything else
+ * is the generated shape.
+ *
+ * The spec is stricter than the API. It declares `normalized_licenses`, `purl` and
+ * `keywords_array` required and non-null; they are not always sent. Every runtime guard
+ * below stays regardless of what these types claim.
+ */
+export type ApiPackage = Loose<Omit<PackageWithRegistry, "repo_metadata" | "advisories">> & Pick<PackageWithRegistry, "id" | "ecosystem" | "name"> & {
+    advisories?: (ApiAdvisory | null)[] | null;
+    repo_metadata?: ApiRepoMetadata | null;
+};
+/** The single `build_info` row, written at the end of every build. */
+export interface BuildInfo {
+    built_at: string;
+    package_count: number;
+    version_count: number;
+    advisory_count: number;
+}
+export type ProgressReporter = (message: string) => void;
+export interface BuildOptions {
+    dbPath?: string;
+    fetchVersionsData?: boolean;
+    onProgress?: ProgressReporter;
+    /**
+     * The API client. Defaults to the shared one; a substitute lets the tests drive the
+     * fetch loop's failure accounting without a network.
+     */
+    client?: Pick<EcosystemsClient, "listCriticalPackages" | "getVersionNumbers">;
+}
+declare function extractDatabase(source: string, destination: string): Promise<void>;
+declare function registryFor(pkg: {
+    ecosystem?: string | null;
+    purl?: string | null;
+}): string | null;
+declare function fetchAllCriticalPackages(onProgress?: ProgressReporter, api?: Pick<EcosystemsClient, "listCriticalPackages">): Promise<ApiPackage[]>;
+/** Version numbers for a package, or `[]` when there is no registry to ask. Throws on
+ * transport failure -- the caller decides whether one package's loss is tolerable. */
+declare function fetchVersionNumbers(ecosystem: string, name: string): Promise<string[]>;
+declare function createDatabase(dbPath: string): DatabaseSync;
+declare function insertPackage(db: DatabaseSync, pkg: ApiPackage): number;
+declare function insertRepoMetadata(db: DatabaseSync, packageId: number, repoMetadata: ApiRepoMetadata | null | undefined, host?: ApiHost | null): void;
+declare function insertAdvisories(db: DatabaseSync, packageId: number, advisories: (ApiAdvisory | null | undefined)[] | null | undefined): void;
+declare function insertVersions(db: DatabaseSync, packageId: number, versionNumbers: string[] | null | undefined): void;
+declare function build(options?: BuildOptions): Promise<BuildInfo>;
+export { build, createDatabase, databasePath, extractDatabase, fetchAllCriticalPackages, fetchVersionNumbers, insertAdvisories, insertPackage, insertRepoMetadata, insertVersions, registryFor, };
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../lib/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAI3C,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EAIzB,MAAM,4BAA4B,CAAC;AAWpC,QAAA,MAAM,YAAY,QAAgD,CAAC;AAGnE;;;;;;GAMG;AACH,KAAK,KAAK,CAAC,CAAC,IAAI;KAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI;CAAE,CAAC;AAEjD;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,QAAQ,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAC1B,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IACtB,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;CACvB;AAED,2DAA2D;AAC3D,MAAM,WAAW,OAAO;IACtB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED;;;;;;;GAOG;AACH,KAAK,eAAe,GAAG,WAAW,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AAC9E,MAAM,MAAM,WAAW,GAAG,KAAK,CAAC,eAAe,CAAC,CAAC;AAEjD;;;;;;;;;;GAUG;AACH,MAAM,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,GAAG,YAAY,CAAC,CAAC,GACvF,IAAI,CAAC,mBAAmB,EAAE,IAAI,GAAG,WAAW,GAAG,MAAM,CAAC,GAAG;IACvD,UAAU,CAAC,EAAE,CAAC,WAAW,GAAG,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;IAC3C,aAAa,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;CACxC,CAAC;AAEJ,sEAAsE;AACtE,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;AAEzD,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B;;;OAGG;IACH,MAAM,CAAC,EAAE,IAAI,CAAC,gBAAgB,EAAE,sBAAsB,GAAG,mBAAmB,CAAC,CAAC;CAC/E;AAOD,iBAAe,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAWjF;AAoDD,iBAAS,WAAW,CAAC,GAAG,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,MAAM,GAAG,IAAI,CAiB5F;AAED,iBAAe,wBAAwB,CACrC,UAAU,CAAC,EAAE,gBAAgB,EAC7B,GAAG,GAAE,IAAI,CAAC,gBAAgB,EAAE,sBAAsB,CAAsB,GACvE,OAAO,CAAC,UAAU,EAAE,CAAC,CAIvB;AAED;sFACsF;AACtF,iBAAe,mBAAmB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAIrF;AAED,iBAAS,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAiHpD;AAED,iBAAS,aAAa,CAAC,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,UAAU,GAAG,MAAM,CA4ChE;AAED,iBAAS,kBAAkB,CACzB,EAAE,EAAE,YAAY,EAChB,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,eAAe,GAAG,IAAI,GAAG,SAAS,EAChD,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,GACpB,IAAI,CA2BN;AAED,iBAAS,gBAAgB,CACvB,EAAE,EAAE,YAAY,EAChB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,CAAC,WAAW,GAAG,IAAI,GAAG,SAAS,CAAC,EAAE,GAAG,IAAI,GAAG,SAAS,GAChE,IAAI,CAsBN;AAED,iBAAS,cAAc,CACrB,EAAE,EAAE,YAAY,EAChB,SAAS,EAAE,MAAM,EACjB,cAAc,EAAE,MAAM,EAAE,GAAG,IAAI,GAAG,SAAS,GAC1C,IAAI,CAUN;AAoBD,iBAAe,KAAK,CAAC,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,SAAS,CAAC,CAiHnE;AAED,OAAO,EACL,KAAK,EACL,cAAc,EACd,YAAY,EACZ,eAAe,EACf,wBAAwB,EACxB,mBAAmB,EACnB,gBAAgB,EAChB,aAAa,EACb,kBAAkB,EAClB,cAAc,EACd,WAAW,GACZ,CAAC"}

package/dist/index.js

@@ -0,0 +1,386 @@
+import { randomBytes } from "node:crypto";
+import { createReadStream, createWriteStream, existsSync } from "node:fs";
+import { mkdir, readFile, rename, unlink } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { pipeline } from "node:stream/promises";
+import { fileURLToPath } from "node:url";
+import { createGunzip } from "node:zlib";
+import { createEcosystemsClient, PURL_TYPE_TO_REGISTRY, parsePurl, purlToRegistry, } from "@ecosyste-ms/ecosystems-ts";
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const pkg = JSON.parse(await readFile(join(__dirname, "..", "package.json"), "utf8"));
+const databasePath = join(__dirname, "..", "critical-packages.db");
+const gzPath = join(__dirname, "..", "critical-packages.db.gz");
+// Extract via a uniquely named temp file, then rename into place. rename(2) is atomic
+// within a filesystem, so concurrent importers see either no database or a complete
+// one. Streaming straight to databasePath let them interleave and produce a file
+// SQLite rejects with "database disk image is malformed" -- permanently, since the
+// corrupt file then satisfies the existsSync check on every subsequent run.
+async function extractDatabase(source, destination) {
+    const tmpPath = `${destination}.${process.pid}-${randomBytes(6).toString("hex")}.tmp`;
+    try {
+        await pipeline(createReadStream(source), createGunzip(), createWriteStream(tmpPath));
+        await rename(tmpPath, destination);
+    }
+    catch (err) {
+        await unlink(tmpPath).catch(() => { });
+        // Another importer finished first and holds the destination open; on Windows that
+        // makes the rename fail. A complete database is there either way, so this is a win.
+        if (!existsSync(destination))
+            throw err;
+    }
+}
+if (!existsSync(databasePath) && existsSync(gzPath)) {
+    await extractDatabase(gzPath, databasePath);
+}
+const CONCURRENCY = 10;
+/** Above this share of failed version fetches the build throws instead of publishing. */
+const MAX_VERSION_FAILURE_RATE = 0.05;
+/**
+ * The mailto that moves the build off the anonymous rate-limit tier.
+ *
+ * Anonymous is 5,000 requests/hour and a full build makes ~9,700, so an unidentified
+ * build gets throttled partway through. With a contact address the tier is "polite",
+ * 15,000.
+ */
+const CONTACT_EMAIL = process.env.ECOSYSTEMS_MAILTO?.trim() || undefined;
+let client;
+/** The SDK client, created once. Owns retries, backoff, 429/Retry-After and pagination. */
+function ecosystemsClient() {
+    if (!client) {
+        client = createEcosystemsClient({
+            userAgent: `${pkg.name}/${pkg.version}`,
+            from: CONTACT_EMAIL,
+        });
+    }
+    return client;
+}
+/**
+ * The ecosyste.ms registry for a package, or null when there is nothing to query.
+ *
+ * Routed through the PURL where there is one: the SDK's table is keyed by PURL type, and
+ * that is the vocabulary the API sends.
+ *
+ * `ecosystem` is the fallback for a package with no PURL, translated through
+ * `ECOSYSTEM_TO_PURL_TYPE` first -- the two vocabularies disagree for four of the
+ * ecosystems here (`rubygems` vs `gem`, `go` vs `golang`, `packagist` vs `composer`,
+ * `homebrew` vs `brew`), and an unresolved registry means zero versions fetched with the
+ * build still reporting success.
+ */
+const ECOSYSTEM_TO_PURL_TYPE = {
+    rubygems: "gem",
+    go: "golang",
+    packagist: "composer",
+    homebrew: "brew",
+};
+function registryFor(pkg) {
+    if (pkg.purl) {
+        try {
+            return purlToRegistry(parsePurl(pkg.purl));
+        }
+        catch {
+            // Malformed PURL: fall through to the ecosystem string.
+        }
+    }
+    const ecosystem = pkg.ecosystem?.toLowerCase();
+    if (!ecosystem)
+        return null;
+    const key = ECOSYSTEM_TO_PURL_TYPE[ecosystem] ?? ecosystem;
+    // Object.hasOwn, not a bare lookup: "constructor" is a valid key and would otherwise
+    // return Object itself -- truthy, typed string, and interpolated into a URL path.
+    const registry = Object.hasOwn(PURL_TYPE_TO_REGISTRY, key)
+        ? PURL_TYPE_TO_REGISTRY[key]
+        : undefined;
+    return registry || null;
+}
+async function fetchAllCriticalPackages(onProgress, api = ecosystemsClient()) {
+    onProgress?.("Fetching critical packages...");
+    const packages = await api.listCriticalPackages();
+    return packages;
+}
+/** Version numbers for a package, or `[]` when there is no registry to ask. Throws on
+ * transport failure -- the caller decides whether one package's loss is tolerable. */
+async function fetchVersionNumbers(ecosystem, name) {
+    const registry = registryFor({ ecosystem });
+    if (!registry)
+        return [];
+    return await ecosystemsClient().getVersionNumbers(registry, name);
+}
+function createDatabase(dbPath) {
+    const db = new DatabaseSync(dbPath);
+    db.exec("PRAGMA journal_mode = WAL");
+    db.exec(`
+    CREATE TABLE packages (
+      id INTEGER PRIMARY KEY,
+      ecosystem TEXT NOT NULL,
+      name TEXT NOT NULL,
+      purl TEXT,
+      namespace TEXT,
+      description TEXT,
+      homepage TEXT,
+      repository_url TEXT,
+      licenses TEXT,
+      normalized_licenses TEXT,
+      latest_version TEXT,
+      versions_count INTEGER,
+      downloads INTEGER,
+      downloads_period TEXT,
+      dependent_packages_count INTEGER,
+      dependent_repos_count INTEGER,
+      first_release_at TEXT,
+      latest_release_at TEXT,
+      last_synced_at TEXT,
+      keywords TEXT,
+      created_at TEXT DEFAULT CURRENT_TIMESTAMP
+    );
+
+    CREATE UNIQUE INDEX idx_packages_ecosystem_name ON packages(ecosystem, name);
+    CREATE INDEX idx_packages_purl ON packages(purl);
+    CREATE INDEX idx_packages_licenses ON packages(licenses);
+    CREATE INDEX idx_packages_ecosystem ON packages(ecosystem);
+
+    CREATE TABLE versions (
+      package_id INTEGER NOT NULL,
+      number TEXT NOT NULL,
+      PRIMARY KEY (package_id, number),
+      FOREIGN KEY (package_id) REFERENCES packages(id)
+    );
+
+    CREATE TABLE advisories (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      package_id INTEGER NOT NULL,
+      uuid TEXT NOT NULL,
+      url TEXT,
+      title TEXT,
+      description TEXT,
+      severity TEXT,
+      published_at TEXT,
+      cvss_score REAL,
+      FOREIGN KEY (package_id) REFERENCES packages(id)
+    );
+
+    CREATE INDEX idx_advisories_package_id ON advisories(package_id);
+    CREATE INDEX idx_advisories_uuid ON advisories(uuid);
+    CREATE INDEX idx_advisories_severity ON advisories(severity);
+    CREATE UNIQUE INDEX idx_advisories_package_uuid ON advisories(package_id, uuid);
+
+    CREATE TABLE repo_metadata (
+      package_id INTEGER PRIMARY KEY,
+      owner TEXT,
+      repo_name TEXT,
+      full_name TEXT,
+      host TEXT,
+      language TEXT,
+      stargazers_count INTEGER,
+      forks_count INTEGER,
+      open_issues_count INTEGER,
+      archived INTEGER,
+      fork INTEGER,
+      FOREIGN KEY (package_id) REFERENCES packages(id)
+    );
+
+    CREATE INDEX idx_repo_full_name ON repo_metadata(full_name);
+    CREATE INDEX idx_repo_owner ON repo_metadata(owner);
+
+    CREATE VIRTUAL TABLE packages_fts USING fts5(
+      ecosystem,
+      name,
+      description,
+      keywords,
+      content=packages,
+      content_rowid=id
+    );
+
+    CREATE TRIGGER packages_ai AFTER INSERT ON packages BEGIN
+      INSERT INTO packages_fts(rowid, ecosystem, name, description, keywords)
+      VALUES (new.id, new.ecosystem, new.name, new.description, new.keywords);
+    END;
+
+    CREATE TRIGGER packages_ad AFTER DELETE ON packages BEGIN
+      INSERT INTO packages_fts(packages_fts, rowid, ecosystem, name, description, keywords)
+      VALUES ('delete', old.id, old.ecosystem, old.name, old.description, old.keywords);
+    END;
+
+    CREATE TRIGGER packages_au AFTER UPDATE ON packages BEGIN
+      INSERT INTO packages_fts(packages_fts, rowid, ecosystem, name, description, keywords)
+      VALUES ('delete', old.id, old.ecosystem, old.name, old.description, old.keywords);
+      INSERT INTO packages_fts(rowid, ecosystem, name, description, keywords)
+      VALUES (new.id, new.ecosystem, new.name, new.description, new.keywords);
+    END;
+
+    CREATE TABLE build_info (
+      id INTEGER PRIMARY KEY CHECK (id = 1),
+      built_at TEXT NOT NULL,
+      package_count INTEGER,
+      version_count INTEGER,
+      advisory_count INTEGER
+    );
+  `);
+    return db;
+}
+function insertPackage(db, pkg) {
+    const stmt = db.prepare(`
+    INSERT OR REPLACE INTO packages (
+      id, ecosystem, name, purl, namespace, description, homepage,
+      repository_url, licenses, normalized_licenses, latest_version,
+      versions_count, downloads, downloads_period, dependent_packages_count,
+      dependent_repos_count, first_release_at, latest_release_at,
+      last_synced_at, keywords
+    ) VALUES (
+      ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+    )
+  `);
+    const keywords = Array.isArray(pkg.keywords_array) ? pkg.keywords_array.join(" ") : null;
+    // Not JSON.stringify alone: it returns undefined for undefined, which node:sqlite
+    // refuses to bind, and the literal string "null" for null.
+    const normalizedLicenses = Array.isArray(pkg.normalized_licenses)
+        ? JSON.stringify(pkg.normalized_licenses)
+        : null;
+    stmt.run(pkg.id, pkg.ecosystem, pkg.name, pkg.purl ?? null, pkg.namespace ?? null, pkg.description ?? null, pkg.homepage ?? null, pkg.repository_url ?? null, pkg.licenses ?? null, normalizedLicenses, pkg.latest_release_number ?? null, pkg.versions_count ?? null, pkg.downloads ?? null, pkg.downloads_period ?? null, pkg.dependent_packages_count ?? null, pkg.dependent_repos_count ?? null, pkg.first_release_published_at ?? null, pkg.latest_release_published_at ?? null, pkg.last_synced_at ?? null, keywords);
+    return pkg.id;
+}
+function insertRepoMetadata(db, packageId, repoMetadata, host) {
+    if (!repoMetadata)
+        return;
+    // The explicit argument stays for callers that have a host from somewhere else.
+    const hostName = host?.name ?? repoMetadata.host?.name ?? null;
+    const repoName = repoMetadata.name ?? repoMetadata.full_name?.split("/").pop() ?? null;
+    const stmt = db.prepare(`
+    INSERT OR REPLACE INTO repo_metadata (
+      package_id, owner, repo_name, full_name, host, language,
+      stargazers_count, forks_count, open_issues_count, archived, fork
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+    stmt.run(packageId, repoMetadata.owner ?? null, repoName, repoMetadata.full_name ?? null, hostName, repoMetadata.language ?? null, repoMetadata.stargazers_count ?? null, repoMetadata.forks_count ?? null, repoMetadata.open_issues_count ?? null, repoMetadata.archived ? 1 : 0, repoMetadata.fork ? 1 : 0);
+}
+function insertAdvisories(db, packageId, advisories) {
+    if (!advisories || advisories.length === 0)
+        return;
+    const stmt = db.prepare(`
+    INSERT OR REPLACE INTO advisories (
+      package_id, uuid, url, title, description, severity, published_at, cvss_score
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+    for (const advisory of advisories) {
+        if (!advisory?.uuid)
+            continue;
+        stmt.run(packageId, advisory.uuid, advisory.url ?? null, advisory.title ?? null, advisory.description ?? null, advisory.severity ?? null, advisory.published_at ?? null, advisory.cvss_score ?? null);
+    }
+}
+function insertVersions(db, packageId, versionNumbers) {
+    if (!versionNumbers || versionNumbers.length === 0)
+        return;
+    const stmt = db.prepare(`
+    INSERT OR REPLACE INTO versions (package_id, number) VALUES (?, ?)
+  `);
+    for (const number of versionNumbers) {
+        stmt.run(packageId, number);
+    }
+}
+/** `SELECT COUNT(*)`, narrowed: node:sqlite hands back `Record<string, SQLOutputValue>`. */
+function countRows(db, table) {
+    const row = db.prepare(`SELECT COUNT(*) as count FROM ${table}`).get();
+    return Number(row?.count ?? 0);
+}
+function updateBuildInfo(db) {
+    db.prepare(`
+    INSERT OR REPLACE INTO build_info (id, built_at, package_count, version_count, advisory_count)
+    VALUES (1, ?, ?, ?, ?)
+  `).run(new Date().toISOString(), countRows(db, "packages"), countRows(db, "versions"), countRows(db, "advisories"));
+}
+async function build(options = {}) {
+    const { dbPath = "critical-packages.db", fetchVersionsData = true, onProgress = console.log, client: api = ecosystemsClient(), } = options;
+    await mkdir(dirname(dbPath) || ".", { recursive: true }).catch(() => { });
+    await unlink(dbPath).catch(() => { });
+    await unlink(`${dbPath}-wal`).catch(() => { });
+    await unlink(`${dbPath}-shm`).catch(() => { });
+    onProgress("Creating database...");
+    const db = createDatabase(dbPath);
+    // Otherwise every throw below leaks the handle, which on Windows locks the file the
+    // caller is about to clean up. The version-failure threshold makes that a reachable
+    // path, not just a corrupt-data one.
+    try {
+        onProgress("Fetching critical packages...");
+        const packages = await fetchAllCriticalPackages(onProgress, api);
+        onProgress(`Found ${packages.length} critical packages`);
+        onProgress("Inserting packages...");
+        db.exec("BEGIN");
+        try {
+            for (const pkg of packages) {
+                insertPackage(db, pkg);
+                insertRepoMetadata(db, pkg.id, pkg.repo_metadata);
+                insertAdvisories(db, pkg.id, pkg.advisories);
+            }
+            db.exec("COMMIT");
+        }
+        catch (err) {
+            db.exec("ROLLBACK");
+            throw err;
+        }
+        if (fetchVersionsData) {
+            onProgress("Fetching versions...");
+            let completed = 0;
+            const total = packages.length;
+            // One package failing is tolerable; a pattern of failures means the run was
+            // throttled. Count them and report at the end rather than swallowing each one.
+            let failed = 0;
+            let firstError;
+            const processPackage = async (pkg) => {
+                const registry = registryFor(pkg);
+                if (!registry)
+                    return { pkg, versions: [] };
+                try {
+                    const versions = await api.getVersionNumbers(registry, pkg.name);
+                    return { pkg, versions };
+                }
+                catch (err) {
+                    failed++;
+                    firstError ??= err;
+                    return { pkg, versions: [] };
+                }
+            };
+            // Process in batches with concurrency limit
+            for (let i = 0; i < packages.length; i += CONCURRENCY) {
+                const batch = packages.slice(i, i + CONCURRENCY);
+                const results = await Promise.all(batch.map(processPackage));
+                db.exec("BEGIN");
+                try {
+                    for (const { pkg, versions } of results) {
+                        if (versions.length > 0) {
+                            insertVersions(db, pkg.id, versions);
+                        }
+                    }
+                    db.exec("COMMIT");
+                }
+                catch (err) {
+                    db.exec("ROLLBACK");
+                    throw err;
+                }
+                completed += batch.length;
+                onProgress(`Fetched versions for ${completed}/${total} packages`);
+            }
+            if (failed > 0) {
+                const share = ((failed / total) * 100).toFixed(1);
+                const reason = firstError instanceof Error ? firstError.message : String(firstError);
+                onProgress(`WARNING: version fetch failed for ${failed}/${total} packages (${share}%)`);
+                onProgress(`  first failure: ${reason}`);
+                // A run that loses this much is not a build worth publishing.
+                if (failed / total > MAX_VERSION_FAILURE_RATE) {
+                    throw new Error(`version fetch failed for ${share}% of packages (limit ${(MAX_VERSION_FAILURE_RATE * 100).toFixed(0)}%); first failure: ${reason}`);
+                }
+            }
+        }
+        updateBuildInfo(db);
+        const info = db
+            .prepare("SELECT * FROM build_info WHERE id = 1")
+            .get();
+        onProgress(`Build complete: ${info.package_count} packages, ${info.version_count} versions, ${info.advisory_count} advisories`);
+        return info;
+    }
+    finally {
+        // The success path returns through here too, and close() is not idempotent.
+        db.close();
+    }
+}
+export { build, createDatabase, databasePath, extractDatabase, fetchAllCriticalPackages, fetchVersionNumbers, insertAdvisories, insertPackage, insertRepoMetadata, insertVersions, registryFor, };
+//# sourceMappingURL=index.js.map