@absolutejs/dataset-gleif 0.0.2 → 0.0.3

package/README.md

@@ -30,5 +30,28 @@ B.V."`. Treat a hit as *"a legal entity by this name"*, not a certainty. It also
 has no industry, headcount, or contacts — pair it with `@absolutejs/discover`
 (people) and `@absolutejs/enrich` (emails). Defaults to ACTIVE entities only.
 
+## Offline snapshot — no rate limits, no network
+
+For high-volume or offline use, build a local SQLite index from GLEIF's full
+**Golden Copy** dump (~2.7M entities) and point the adapter at it:
+
+1. **Download** the Golden Copy (LEI2, CSV or `.zip`) from
+   <https://goldencopy.gleif.org> (it's CC0). Their bulk host WAF-blocks many
+   data-center IPs, so fetch it from a browser / allowed network — this tool does
+   not download it for you.
+2. **Build** the snapshot:
+   ```sh
+   bunx gleif-snapshot golden-copy.zip gleif-snapshot.sqlite
+   # or: bun run node_modules/@absolutejs/dataset-gleif/dist/snapshot.js <file> [out]
+   ```
+3. **Use** it:
+   ```ts
+   gleifSource({ snapshotPath: "gleif-snapshot.sqlite" });
+   ```
+   `findCompany` resolves from the snapshot — instant, offline, zero rate limits —
+   falling back to the live API only on a miss. Re-run the generator to refresh
+   (GLEIF republishes 3×/day). Snapshot reads use `bun:sqlite` (loaded lazily;
+   live-only use needs nothing).
+
 Apache-2.0. Pure importer of public CC0 data — the self-collected, self-healing
 dataset is deliberately not part of any adapter.

package/dist/index.d.ts

@@ -1,9 +1,14 @@
 import type { DatasetSource } from "@absolutejs/discover";
+export declare const normalizeCompanyName: (value: string) => string;
 export type GleifOptions = {
     /** Inject a fetch (proxy, test stub). Defaults to global fetch. */
     fetch?: typeof fetch;
     timeoutMs?: number;
     /** Only consider ACTIVE legal entities (default true). */
     activeOnly?: boolean;
+    /** Path to a SQLite snapshot built by the snapshot generator (see scripts/).
+     *  When set, findCompany resolves from it first — instant, offline, no rate
+     *  limit — and falls back to the live API only on a miss. */
+    snapshotPath?: string;
 };
 export declare const gleifSource: (options?: GleifOptions) => DatasetSource;

package/dist/index.js

@@ -1,8 +1,23 @@
 // @bun
+var __require = import.meta.require;
+
 // src/index.ts
 var GLEIF_API = "https://api.gleif.org/api/v1";
 var DEFAULT_TIMEOUT_MS = 1e4;
 var DEFAULT_PAGE_SIZE = 10;
+var normalizeCompanyName = (value) => value.toLowerCase().normalize("NFKD").replace(/[^\p{L}\p{N} ]/gu, " ").replace(/\b(inc|incorporated|corp|corporation|llc|llp|lp|ltd|limited|co|company|gmbh|ag|sa|bv|nv|plc|holding|holdings|group|the)\b/g, " ").replace(/\s+/g, " ").trim();
+var snapshotLookups = new Map;
+var getSnapshotLookup = async (path) => {
+  const existing = snapshotLookups.get(path);
+  if (existing)
+    return existing;
+  const { Database } = await import("bun:sqlite");
+  const db = new Database(path, { readonly: true });
+  const statement = db.query("SELECT lei, name, country, status FROM company WHERE norm_name = ?1 ORDER BY CASE status WHEN 'ACTIVE' THEN 0 ELSE 1 END LIMIT 1");
+  const lookup = (norm) => statement.get(norm);
+  snapshotLookups.set(path, lookup);
+  return lookup;
+};
 var toCompany = (record) => {
   const entity = record.attributes?.entity;
   const name = entity?.legalName?.name?.trim();
@@ -25,6 +40,20 @@ var gleifSource = (options = {}) => {
     const query = name?.trim();
     if (!query)
       return null;
+    if (options.snapshotPath) {
+      try {
+        const lookup = await getSnapshotLookup(options.snapshotPath);
+        const row = lookup(normalizeCompanyName(query));
+        if (row) {
+          return {
+            country: row.country ?? undefined,
+            name: row.name,
+            registryId: row.lei,
+            source: "gleif"
+          };
+        }
+      } catch {}
+    }
     const params = new URLSearchParams;
     params.set("filter[fulltext]", query);
     params.set("page[size]", String(DEFAULT_PAGE_SIZE));
@@ -50,5 +79,6 @@ var gleifSource = (options = {}) => {
   return { findCompany, name: "gleif" };
 };
 export {
+  normalizeCompanyName,
   gleifSource
 };

package/dist/snapshot.js

@@ -0,0 +1,132 @@
+#!/usr/bin/env bun
+// @bun
+
+// scripts/snapshot.ts
+import { Database } from "bun:sqlite";
+
+// src/index.ts
+var normalizeCompanyName = (value) => value.toLowerCase().normalize("NFKD").replace(/[^\p{L}\p{N} ]/gu, " ").replace(/\b(inc|incorporated|corp|corporation|llc|llp|lp|ltd|limited|co|company|gmbh|ag|sa|bv|nv|plc|holding|holdings|group|the)\b/g, " ").replace(/\s+/g, " ").trim();
+var snapshotLookups = new Map;
+
+// scripts/snapshot.ts
+var COLUMNS = {
+  country: "Entity.LegalAddress.Country",
+  lei: "LEI",
+  name: "Entity.LegalName",
+  status: "Entity.EntityStatus"
+};
+var BATCH_SIZE = 1e4;
+var PROGRESS_EVERY = 1e5;
+async function* csvRows(stream) {
+  const decoder = new TextDecoder;
+  let field = "";
+  let row = [];
+  let inQuotes = false;
+  let quotePending = false;
+  for await (const chunk of stream) {
+    const text = decoder.decode(chunk, { stream: true });
+    for (const char of text) {
+      if (quotePending) {
+        quotePending = false;
+        if (char === '"') {
+          field += '"';
+          inQuotes = true;
+          continue;
+        }
+        inQuotes = false;
+      }
+      if (inQuotes) {
+        if (char === '"')
+          quotePending = true;
+        else
+          field += char;
+      } else if (char === '"') {
+        inQuotes = true;
+      } else if (char === ",") {
+        row.push(field);
+        field = "";
+      } else if (char === `
+`) {
+        row.push(field);
+        yield row;
+        field = "";
+        row = [];
+      } else if (char !== "\r") {
+        field += char;
+      }
+    }
+  }
+  if (field.length > 0 || row.length > 0) {
+    row.push(field);
+    yield row;
+  }
+}
+var inputPath = process.argv[2];
+var outputPath = process.argv[3] ?? "gleif-snapshot.sqlite";
+if (!inputPath) {
+  console.error("usage: bun run scripts/snapshot.ts <golden-copy.csv|.zip> [out.sqlite]");
+  process.exit(1);
+}
+var source = inputPath.endsWith(".zip") ? Bun.spawn(["unzip", "-p", inputPath]).stdout : Bun.file(inputPath).stream();
+var db = new Database(outputPath, { create: true });
+db.run("PRAGMA journal_mode = WAL");
+db.run("DROP TABLE IF EXISTS company");
+db.run("CREATE TABLE company (lei TEXT PRIMARY KEY, name TEXT NOT NULL, norm_name TEXT NOT NULL, country TEXT, status TEXT)");
+var insert = db.query("INSERT OR REPLACE INTO company (lei, name, norm_name, country, status) VALUES (?1, ?2, ?3, ?4, ?5)");
+var insertBatch = db.transaction((batch) => {
+  for (const values of batch)
+    insert.run(...values);
+});
+var cols = null;
+var batch = [];
+var total = 0;
+var flush = () => {
+  if (batch.length > 0) {
+    insertBatch(batch);
+    batch = [];
+  }
+};
+for await (const row of csvRows(source)) {
+  if (!cols) {
+    const map = {};
+    row.forEach((column, index) => {
+      map[column.replace(/^\uFEFF/, "").trim()] = index;
+    });
+    const lei2 = map[COLUMNS.lei];
+    const name2 = map[COLUMNS.name];
+    if (lei2 === undefined || name2 === undefined) {
+      console.error(`Missing required columns. Found headers: ${row.slice(0, 8).join(", ")}\u2026`);
+      process.exit(1);
+    }
+    cols = {
+      country: map[COLUMNS.country] ?? -1,
+      lei: lei2,
+      name: name2,
+      status: map[COLUMNS.status] ?? -1
+    };
+    continue;
+  }
+  const lei = row[cols.lei]?.trim();
+  const name = row[cols.name]?.trim();
+  if (!lei || !name)
+    continue;
+  batch.push([
+    lei,
+    name,
+    normalizeCompanyName(name),
+    cols.country >= 0 ? row[cols.country]?.trim() ?? "" : "",
+    cols.status >= 0 ? row[cols.status]?.trim() ?? "" : ""
+  ]);
+  total += 1;
+  if (batch.length >= BATCH_SIZE)
+    flush();
+  if (total % PROGRESS_EVERY === 0) {
+    console.log(`  ${total.toLocaleString()} rows\u2026`);
+  }
+}
+flush();
+console.log("Indexing\u2026");
+db.run("CREATE INDEX idx_company_norm_name ON company(norm_name)");
+db.run("PRAGMA wal_checkpoint(TRUNCATE)");
+db.close();
+console.log(`Done \u2014 ${total.toLocaleString()} entities \u2192 ${outputPath}`);

package/package.json

@@ -20,6 +20,9 @@
     "lei",
     "company-data"
   ],
+  "bin": {
+    "gleif-snapshot": "./dist/snapshot.js"
+  },
   "license": "Apache-2.0",
   "main": "./dist/index.js",
   "name": "@absolutejs/dataset-gleif",
@@ -35,11 +38,11 @@
     "url": "https://github.com/absolutejs/dataset-adapters.git"
   },
   "scripts": {
-    "build": "rm -rf dist && bun build src/index.ts --outdir dist --target=bun --external @absolutejs/discover && tsc --emitDeclarationOnly --project tsconfig.json",
+    "build": "rm -rf dist && bun build src/index.ts --outdir dist --target=bun --external @absolutejs/discover && bun build scripts/snapshot.ts --outdir dist --target=bun --external @absolutejs/discover && tsc --emitDeclarationOnly --project tsconfig.json",
     "release": "bun run build && bun publish",
     "typecheck": "tsc --noEmit"
   },
   "type": "module",
   "types": "./dist/index.d.ts",
-  "version": "0.0.2"
+  "version": "0.0.3"
 }