@pgsage/loaders 0.1.0

package/README.md

@@ -0,0 +1,78 @@
+# @pgsage/loaders
+
+Data ingestion pipelines for pgsage. Currently ships the Census ACS 5-year estimates loader.
+
+---
+
+## Running
+
+```sh
+# Full load — all 50 states + DC + PR (~3–5 minutes)
+pnpm --filter @pgsage/loaders load:census
+
+# Smoke test — Rhode Island only (~10 seconds)
+pnpm --filter @pgsage/loaders load:census:smoke
+```
+
+Requires `DATABASE_URL`, `VOYAGE_API_KEY`, and optionally `CENSUS_API_KEY` in `.env`.
+
+---
+
+## What Gets Loaded
+
+The Census ACS loader fetches from the [Census Bureau API](https://api.census.gov/data.html) and populates three tables:
+
+| Table | Rows | Description |
+|-------|------|-------------|
+| `public.geography` | 3,274 | States + counties with FIPS geo IDs |
+| `public.variables` | 247 | ACS variable codes, labels, and categories |
+| `public.estimates` | ~808,678 | Estimate values + margins of error |
+
+After loading, the schema is indexed into `public.schema_embeddings` using Voyage AI `voyage-4-lite` embeddings (1024 dimensions) for pgvector similarity search.
+
+### Variable categories
+
+| Category | Examples |
+|----------|---------|
+| `income` | Median household income, per-capita income, Gini index |
+| `poverty` | Population below poverty line, poverty ratios |
+| `housing` | Home values, gross rent, occupancy, homeownership |
+| `employment` | Labor force participation, unemployment, industry |
+| `demographics` | Total population, age, sex, race, Hispanic origin |
+| `education` | High school, bachelor's, graduate degree attainment |
+| `transportation` | Commute mode, work from home, travel time |
+| `health_insurance` | Coverage by age and sex |
+
+---
+
+## Schema Indexing
+
+After data load, `indexSchema()` from `@pgsage/core` generates natural-language descriptions of each table and column, embeds them with Voyage AI, and upserts into `schema_embeddings`. This enables the retriever to find relevant schema context via cosine similarity.
+
+Re-index at any time:
+
+```sh
+pnpm --filter @pgsage/loaders load:census  # re-runs indexing automatically
+```
+
+---
+
+## Adding New Datasets
+
+Each loader follows the same pattern:
+
+1. Fetch data from the source API
+2. Upsert into `geography`, `variables`, and `estimates` tables
+3. Call `indexSchema()` to update embeddings
+
+Add a new entry point in `src/<dataset>/index.ts` and a corresponding script in `package.json`. The retriever and planner automatically pick up any new schema content on next query.
+
+---
+
+## Testing
+
+```sh
+pnpm --filter @pgsage/loaders test
+```
+
+51 unit tests covering API client, data transformation, and schema indexing logic.

package/dist/census/api.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Census Bureau ACS 5-Year API client.
+ *
+ * All functions use Node's built-in fetch (Node 22+). The Census API returns
+ * a 2D array of strings: row 0 is the header, subsequent rows are data.
+ * All numeric values come back as strings and must be parsed; null/suppressed
+ * data arrives as the string "null" or "-666666666".
+ */
+import type { CensusConfig, CensusApiResponse, GeoRecord, EstimateRecord, VariableDefinition } from "./types.js";
+/** FIPS code for Rhode Island — used in smoke-test mode. */
+export declare const SMOKE_STATE_FIPS = "44";
+/**
+ * Split an array into chunks of at most `size` elements.
+ */
+export declare function chunk<T>(arr: T[], size: number): T[][];
+/**
+ * Parse a Census API numeric string value.
+ * Returns null for "null", empty string, or the suppressed-value sentinel.
+ */
+export declare function parseCensusNumber(raw: string | null | undefined): number | null;
+/**
+ * Low-level Census API GET with retry on 429 and 5xx.
+ * Returns the parsed 2D string array from the Census JSON response.
+ */
+export declare function censusGet(url: string, config: CensusConfig): Promise<CensusApiResponse>;
+/**
+ * Fetch all states (and DC + PR) from the ACS.
+ * Returns one GeoRecord per state, level='state'.
+ */
+export declare function fetchStates(config: CensusConfig): Promise<GeoRecord[]>;
+/**
+ * Fetch all counties for a given set of state FIPS codes (or all states with '*').
+ * In smoke mode, only fetches counties for Rhode Island (FIPS 44).
+ */
+export declare function fetchCounties(config: CensusConfig, stateFips?: string): Promise<GeoRecord[]>;
+/**
+ * Fetch estimates for a set of variables at either state or county level.
+ *
+ * Batches variables into chunks of MAX_ESTIMATE_VARS_PER_REQUEST to stay
+ * within Census API limits (each _E code is paired with its _M code, so
+ * 24 estimate vars = 48 actual vars in the request, plus NAME = 49 total).
+ *
+ * Inserts a batchDelayMs pause between requests to be polite to the API.
+ */
+export declare function fetchEstimates(config: CensusConfig, geoLevel: "state" | "county", variables: VariableDefinition[], stateFips?: string): Promise<EstimateRecord[]>;
+//# sourceMappingURL=api.d.ts.map

package/dist/census/api.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../../src/census/api.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EACV,YAAY,EACZ,iBAAiB,EACjB,SAAS,EACT,cAAc,EACd,kBAAkB,EACnB,MAAM,YAAY,CAAC;AAYpB,4DAA4D;AAC5D,eAAO,MAAM,gBAAgB,OAAO,CAAC;AAMrC;;GAEG;AACH,wBAAgB,KAAK,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,CAAC,EAAE,EAAE,CAMtD;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAI/E;AAUD;;;GAGG;AACH,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAwC7F;AAaD;;;GAGG;AACH,wBAAsB,WAAW,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAiB5E;AAED;;;GAGG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,YAAY,EACpB,SAAS,GAAE,MAAY,GACtB,OAAO,CAAC,SAAS,EAAE,CAAC,CAsBtB;AAMD;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,YAAY,EACpB,QAAQ,EAAE,OAAO,GAAG,QAAQ,EAC5B,SAAS,EAAE,kBAAkB,EAAE,EAC/B,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,EAAE,CAAC,CA2D3B"}

package/dist/census/api.js

@@ -0,0 +1,204 @@
+/**
+ * Census Bureau ACS 5-Year API client.
+ *
+ * All functions use Node's built-in fetch (Node 22+). The Census API returns
+ * a 2D array of strings: row 0 is the header, subsequent rows are data.
+ * All numeric values come back as strings and must be parsed; null/suppressed
+ * data arrives as the string "null" or "-666666666".
+ */
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/** Census suppression sentinel. Treat as null. */
+const SUPPRESSED_VALUE = "-666666666";
+/** Maximum number of _E variables per request (paired with _M = 50 total vars). */
+const MAX_ESTIMATE_VARS_PER_REQUEST = 24;
+/** FIPS code for Rhode Island — used in smoke-test mode. */
+export const SMOKE_STATE_FIPS = "44";
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+/**
+ * Split an array into chunks of at most `size` elements.
+ */
+export function chunk(arr, size) {
+    const result = [];
+    for (let i = 0; i < arr.length; i += size) {
+        result.push(arr.slice(i, i + size));
+    }
+    return result;
+}
+/**
+ * Parse a Census API numeric string value.
+ * Returns null for "null", empty string, or the suppressed-value sentinel.
+ */
+export function parseCensusNumber(raw) {
+    if (raw == null || raw === "" || raw === "null" || raw === SUPPRESSED_VALUE)
+        return null;
+    const n = Number(raw);
+    return Number.isNaN(n) ? null : n;
+}
+/**
+ * Sleep for `ms` milliseconds. Used to be polite to the Census API between
+ * batch requests.
+ */
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Low-level Census API GET with retry on 429 and 5xx.
+ * Returns the parsed 2D string array from the Census JSON response.
+ */
+export async function censusGet(url, config) {
+    const maxRetries = config.maxRetries ?? 3;
+    const timeoutMs = config.requestTimeoutMs ?? 30_000;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        const signal = AbortSignal.timeout(timeoutMs);
+        let res;
+        try {
+            res = await fetch(url, { signal });
+        }
+        catch (err) {
+            if (attempt < maxRetries) {
+                await sleep(1000 * 2 ** attempt);
+                continue;
+            }
+            throw new Error(`Census API fetch failed after ${maxRetries + 1} attempts: ${err instanceof Error ? err.message : String(err)}`, { cause: err });
+        }
+        if (res.status === 429 || res.status >= 500) {
+            if (attempt < maxRetries) {
+                await sleep(1000 * 2 ** attempt);
+                continue;
+            }
+            throw new Error(`Census API error ${res.status} after ${maxRetries + 1} attempts: ${url}`);
+        }
+        if (!res.ok) {
+            const body = await res.text().catch(() => "(no body)");
+            throw new Error(`Census API error ${res.status}: ${body}`);
+        }
+        const data = (await res.json());
+        return data;
+    }
+    // Unreachable, but TypeScript needs it
+    throw new Error("Census API: exhausted retries");
+}
+/**
+ * Build the base Census API URL for 5-Year ACS estimates.
+ */
+function baseUrl(config) {
+    return `${config.baseUrl}/${config.vintage}/acs/acs5`;
+}
+// ---------------------------------------------------------------------------
+// Geography fetchers
+// ---------------------------------------------------------------------------
+/**
+ * Fetch all states (and DC + PR) from the ACS.
+ * Returns one GeoRecord per state, level='state'.
+ */
+export async function fetchStates(config) {
+    const url = `${baseUrl(config)}?get=NAME&for=state:*&key=${config.apiKey}`;
+    const rows = await censusGet(url, config);
+    // Row 0 is headers: ["NAME", "state"]
+    return rows.slice(1).map((row) => {
+        const [name, stateFips] = row;
+        return {
+            geoId: stateFips,
+            stateFips,
+            countyFips: null,
+            stateName: name,
+            countyName: null,
+            level: "state",
+        };
+    });
+}
+/**
+ * Fetch all counties for a given set of state FIPS codes (or all states with '*').
+ * In smoke mode, only fetches counties for Rhode Island (FIPS 44).
+ */
+export async function fetchCounties(config, stateFips = "*") {
+    const inClause = stateFips === "*" ? "state:*" : `state:${stateFips}`;
+    const url = `${baseUrl(config)}?get=NAME&for=county:*&in=${inClause}&key=${config.apiKey}`;
+    const rows = await censusGet(url, config);
+    // Row 0 is headers: ["NAME", "state", "county"]
+    return rows.slice(1).map((row) => {
+        const [name, sFips, cFips] = row;
+        // NAME is "County Name, State Name" — split on ", " to get county name
+        const commaIdx = name.lastIndexOf(", ");
+        const countyName = commaIdx >= 0 ? name.slice(0, commaIdx) : name;
+        const stateName = commaIdx >= 0 ? name.slice(commaIdx + 2) : "";
+        return {
+            geoId: sFips + cFips, // 5-digit FIPS
+            stateFips: sFips,
+            countyFips: cFips,
+            stateName,
+            countyName,
+            level: "county",
+        };
+    });
+}
+// ---------------------------------------------------------------------------
+// Estimates fetcher
+// ---------------------------------------------------------------------------
+/**
+ * Fetch estimates for a set of variables at either state or county level.
+ *
+ * Batches variables into chunks of MAX_ESTIMATE_VARS_PER_REQUEST to stay
+ * within Census API limits (each _E code is paired with its _M code, so
+ * 24 estimate vars = 48 actual vars in the request, plus NAME = 49 total).
+ *
+ * Inserts a batchDelayMs pause between requests to be polite to the API.
+ */
+export async function fetchEstimates(config, geoLevel, variables, stateFips) {
+    const batchDelay = config.batchDelayMs ?? 200;
+    const batches = chunk(variables, MAX_ESTIMATE_VARS_PER_REQUEST);
+    const results = [];
+    for (let i = 0; i < batches.length; i++) {
+        if (i > 0)
+            await sleep(batchDelay);
+        const batch = batches[i];
+        const eCodes = batch.map((v) => v.code);
+        const mCodes = batch.map((v) => v.code.replace(/E$/, "M"));
+        const allCodes = [...eCodes, ...mCodes];
+        let forClause;
+        let inClause = null;
+        if (geoLevel === "state") {
+            // Restrict to a single state in smoke mode; otherwise fetch all states
+            forClause = stateFips ? `state:${stateFips}` : "state:*";
+        }
+        else {
+            // County — must add &in=state:* or restrict to one state
+            forClause = "county:*";
+            inClause = stateFips ? `state:${stateFips}` : "state:*";
+        }
+        const url = [
+            `${baseUrl(config)}?get=NAME,${allCodes.join(",")}`,
+            `&for=${forClause}`,
+            inClause ? `&in=${inClause}` : "",
+            `&key=${config.apiKey}`,
+        ].join("");
+        const rows = await censusGet(url, config);
+        // Row 0: ["NAME", ...eCodes, ...mCodes, "state"] or ["NAME", ..., "state", "county"]
+        const header = rows[0];
+        const stateIdx = header.lastIndexOf("state");
+        const countyIdx = header.indexOf("county");
+        for (const row of rows.slice(1)) {
+            const sFips = row[stateIdx] ?? "";
+            const cFips = countyIdx >= 0 ? (row[countyIdx] ?? null) : null;
+            const geoId = cFips ? sFips + cFips : sFips;
+            for (let j = 0; j < eCodes.length; j++) {
+                const eIdx = header.indexOf(eCodes[j]);
+                const mIdx = header.indexOf(mCodes[j]);
+                const estimate = parseCensusNumber(eIdx >= 0 ? row[eIdx] : null);
+                const marginError = parseCensusNumber(mIdx >= 0 ? row[mIdx] : null);
+                results.push({
+                    geoId,
+                    variableCode: eCodes[j],
+                    estimate,
+                    marginError,
+                });
+            }
+        }
+    }
+    return results;
+}
+//# sourceMappingURL=api.js.map

package/dist/census/api.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"api.js","sourceRoot":"","sources":["../../src/census/api.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAUH,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,kDAAkD;AAClD,MAAM,gBAAgB,GAAG,YAAY,CAAC;AAEtC,mFAAmF;AACnF,MAAM,6BAA6B,GAAG,EAAE,CAAC;AAEzC,4DAA4D;AAC5D,MAAM,CAAC,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAErC,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;;GAEG;AACH,MAAM,UAAU,KAAK,CAAI,GAAQ,EAAE,IAAY;IAC7C,MAAM,MAAM,GAAU,EAAE,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAA8B;IAC9D,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,KAAK,EAAE,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,gBAAgB;QAAE,OAAO,IAAI,CAAC;IACzF,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;;GAGG;AACH,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,MAAoB;IAC/D,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,MAAM,CAAC,gBAAgB,IAAI,MAAM,CAAC;IAEpD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAE9C,IAAI,GAAa,CAAC;QAClB,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;gBACzB,MAAM,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC;gBACjC,SAAS;YACX,CAAC;YACD,MAAM,IAAI,KAAK,CACb,iCAAiC,UAAU,GAAG,CAAC,cAAc,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC/G,EAAE,KAAK,EAAE,GAAG,EAAE,CACf,CAAC;QACJ,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YAC5C,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;gBACzB,MAAM,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC;gBACjC,SAAS;YACX,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,CAAC,MAAM,UAAU,UAAU,GAAG,CAAC,cAAc,GAAG,EAAE,CAAC,CAAC;QAC7F,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC;YACvD,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAsB,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,uCAAuC;IACvC,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;AACnD,CAAC;AAED;;GAEG;AACH,SAAS,OAAO,CAAC,MAAoB;IACnC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,WAAW,CAAC;AACxD,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAoB;IACpD,MAAM,GAAG,GACP,GAAG,OAAO,CAAC,MAAM,CAAC,6BAA6B,MAAM,CAAC,MAAM,EAAE,CAAC;IAEjE,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC1C,sCAAsC;IACtC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC/B,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,GAAuB,CAAC;QAClD,OAAO;YACL,KAAK,EAAE,SAAS;YAChB,SAAS;YACT,UAAU,EAAE,IAAI;YAChB,SAAS,EAAE,IAAI;YACf,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,OAAgB;SACxB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAoB,EACpB,YAAoB,GAAG;IAEvB,MAAM,QAAQ,GAAG,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,SAAS,EAAE,CAAC;IACtE,MAAM,GAAG,GACP,GAAG,OAAO,CAAC,MAAM,CAAC,6BAA6B,QAAQ,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;IAEjF,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC1C,gDAAgD;IAChD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC/B,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,GAAG,GAA+B,CAAC;QAC7D,uEAAuE;QACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,UAAU,GAAG,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAClE,MAAM,SAAS,GAAG,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,OAAO;YACL,KAAK,EAAE,KAAK,GAAG,KAAK,EAAO,eAAe;YAC1C,SAAS,EAAE,KAAK;YAChB,UAAU,EAAE,KAAK;YACjB,SAAS;YACT,UAAU;YACV,KAAK,EAAE,QAAiB;SACzB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAoB,EACpB,QAA4B,EAC5B,SAA+B,EAC/B,SAAkB;IAElB,MAAM,UAAU,GAAG,MAAM,CAAC,YAAY,IAAI,GAAG,CAAC;IAC9C,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,EAAE,6BAA6B,CAAC,CAAC;IAChE,MAAM,OAAO,GAAqB,EAAE,CAAC;IAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,CAAC,GAAG,CAAC;YAAE,MAAM,KAAK,CAAC,UAAU,CAAC,CAAC;QAEnC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,CAAC,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;QAExC,IAAI,SAAiB,CAAC;QACtB,IAAI,QAAQ,GAAkB,IAAI,CAAC;QAEnC,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YACzB,uEAAuE;YACvE,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3D,CAAC;aAAM,CAAC;YACN,yDAAyD;YACzD,SAAS,GAAG,UAAU,CAAC;YACvB,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1D,CAAC;QAED,MAAM,GAAG,GAAG;YACV,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;YACnD,QAAQ,SAAS,EAAE;YACnB,QAAQ,CAAC,CAAC,CAAC,OAAO,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE;YACjC,QAAQ,MAAM,CAAC,MAAM,EAAE;SACxB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEX,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAC1C,qFAAqF;QACrF,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAE3C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/D,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC;gBACxC,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC;gBACxC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACjE,MAAM,WAAW,GAAG,iBAAiB,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACpE,OAAO,CAAC,IAAI,CAAC;oBACX,KAAK;oBACL,YAAY,EAAE,MAAM,CAAC,CAAC,CAAE;oBACxB,QAAQ;oBACR,WAAW;iBACZ,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/census/index.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Census ACS Loader — CLI entrypoint
+ *
+ * Fetches 2024 ACS 5-Year data from the Census Bureau API and populates
+ * the local pgsage Postgres database, then indexes the schema into
+ * schema_embeddings via Voyage AI.
+ *
+ * Usage:
+ *   pnpm --filter @pgsage/loaders load:census          # full run (~3-5 min)
+ *   pnpm --filter @pgsage/loaders load:census:smoke    # smoke test (~10 sec)
+ *   pnpm --filter @pgsage/loaders load:census -- --smoke
+ *
+ * Required env vars: CENSUS_API_KEY, DATABASE_URL, VOYAGE_API_KEY
+ */
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/census/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/census/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG"}

package/dist/census/index.js

@@ -0,0 +1,122 @@
+/**
+ * Census ACS Loader — CLI entrypoint
+ *
+ * Fetches 2024 ACS 5-Year data from the Census Bureau API and populates
+ * the local pgsage Postgres database, then indexes the schema into
+ * schema_embeddings via Voyage AI.
+ *
+ * Usage:
+ *   pnpm --filter @pgsage/loaders load:census          # full run (~3-5 min)
+ *   pnpm --filter @pgsage/loaders load:census:smoke    # smoke test (~10 sec)
+ *   pnpm --filter @pgsage/loaders load:census -- --smoke
+ *
+ * Required env vars: CENSUS_API_KEY, DATABASE_URL, VOYAGE_API_KEY
+ */
+import pg from "pg";
+import { introspector, retriever, embeddings } from "@pgsage/core";
+const { introspect } = introspector;
+const { indexSchema } = retriever;
+const { VoyageEmbeddingProvider } = embeddings;
+import { fetchStates, fetchCounties, fetchEstimates, SMOKE_STATE_FIPS } from "./api.js";
+import { upsertGeography, upsertVariables, upsertEstimates } from "./loader.js";
+import { CENSUS_VARIABLES, SMOKE_TEST_VARIABLES } from "./variables.js";
+// ---------------------------------------------------------------------------
+// Entry point
+// ---------------------------------------------------------------------------
+const isSmoke = process.argv.includes("--smoke");
+async function run() {
+    const startMs = Date.now();
+    // --- Validate environment -----------------------------------------------
+    const apiKey = process.env["CENSUS_API_KEY"];
+    const databaseUrl = process.env["DATABASE_URL"];
+    const voyageApiKey = process.env["VOYAGE_API_KEY"];
+    if (!apiKey)
+        throw new Error("CENSUS_API_KEY environment variable is required.");
+    if (!databaseUrl)
+        throw new Error("DATABASE_URL environment variable is required.");
+    if (!voyageApiKey)
+        throw new Error("VOYAGE_API_KEY environment variable is required.");
+    const config = {
+        apiKey,
+        vintage: 2024,
+        baseUrl: "https://api.census.gov/data",
+        databaseUrl,
+        voyageApiKey,
+        smoke: isSmoke,
+        batchDelayMs: 200,
+        maxRetries: 3,
+        requestTimeoutMs: 30_000,
+    };
+    const variables = isSmoke ? SMOKE_TEST_VARIABLES : CENSUS_VARIABLES;
+    const mode = isSmoke ? "SMOKE TEST" : "FULL";
+    console.log(`\npgsage Census ACS Loader — ${mode} mode`);
+    console.log(`Vintage: ${config.vintage} ACS 5-Year`);
+    console.log(`Variables: ${variables.length}`);
+    if (isSmoke) {
+        console.log(`Geography: Rhode Island only (state FIPS ${SMOKE_STATE_FIPS})`);
+    }
+    console.log("");
+    // --- Database setup -------------------------------------------------------
+    const pool = new pg.Pool({ connectionString: databaseUrl });
+    const summary = {
+        geographiesUpserted: 0,
+        variablesUpserted: 0,
+        estimatesUpserted: 0,
+        embeddingTokens: 0,
+        elapsedMs: 0,
+    };
+    try {
+        // --- 1. Geography -------------------------------------------------------
+        console.log("Step 1/6: Fetching states...");
+        const states = isSmoke
+            ? (await fetchStates(config)).filter((g) => g.stateFips === SMOKE_STATE_FIPS)
+            : await fetchStates(config);
+        summary.geographiesUpserted += await upsertGeography(pool, states);
+        console.log(`  ✓ ${states.length} states upserted`);
+        console.log("Step 2/6: Fetching counties...");
+        const counties = await fetchCounties(config, isSmoke ? SMOKE_STATE_FIPS : "*");
+        summary.geographiesUpserted += await upsertGeography(pool, counties);
+        console.log(`  ✓ ${counties.length} counties upserted`);
+        // --- 2. Variables -------------------------------------------------------
+        console.log("Step 3/6: Upserting variable definitions...");
+        summary.variablesUpserted = await upsertVariables(pool, variables);
+        console.log(`  ✓ ${summary.variablesUpserted} variables upserted`);
+        // --- 3. Estimates — states ----------------------------------------------
+        console.log("Step 4/6: Fetching state-level estimates...");
+        const stateEstimates = await fetchEstimates(config, "state", variables, isSmoke ? SMOKE_STATE_FIPS : undefined);
+        summary.estimatesUpserted += await upsertEstimates(pool, stateEstimates);
+        console.log(`  ✓ ${stateEstimates.length} state estimate rows upserted`);
+        // --- 4. Estimates — counties --------------------------------------------
+        console.log("Step 5/6: Fetching county-level estimates...");
+        const countyEstimates = await fetchEstimates(config, "county", variables, isSmoke ? SMOKE_STATE_FIPS : undefined);
+        summary.estimatesUpserted += await upsertEstimates(pool, countyEstimates);
+        console.log(`  ✓ ${countyEstimates.length} county estimate rows upserted`);
+        // --- 5. Schema embedding -----------------------------------------------
+        console.log("Step 6/6: Introspecting schema and indexing embeddings...");
+        const schema = await introspect(pool, { schemas: ["public"] });
+        const embedder = new VoyageEmbeddingProvider({ apiKey: voyageApiKey });
+        const indexResult = await indexSchema(pool, schema, embedder);
+        summary.embeddingTokens = indexResult.totalTokens;
+        console.log(`  ✓ ${indexResult.rowsUpserted} schema embedding rows upserted`);
+        console.log(`  ✓ ${indexResult.totalTokens} embedding tokens consumed`);
+        // --- Summary ------------------------------------------------------------
+        summary.elapsedMs = Date.now() - startMs;
+        const elapsed = (summary.elapsedMs / 1000).toFixed(1);
+        console.log(`\n${"─".repeat(50)}`);
+        console.log(`Census load complete in ${elapsed}s`);
+        console.log(`  Geographies : ${summary.geographiesUpserted}`);
+        console.log(`  Variables   : ${summary.variablesUpserted}`);
+        console.log(`  Estimates   : ${summary.estimatesUpserted}`);
+        console.log(`  Emb. tokens : ${summary.embeddingTokens}`);
+        console.log(`${"─".repeat(50)}\n`);
+    }
+    finally {
+        await pool.end();
+    }
+}
+void run().catch((error) => {
+    console.error("\nCensus loader failed:");
+    console.error(error instanceof Error ? error.message : error);
+    process.exitCode = 1;
+});
+//# sourceMappingURL=index.js.map

package/dist/census/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/census/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAEnE,MAAM,EAAE,UAAU,EAAE,GAAG,YAAY,CAAC;AACpC,MAAM,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;AAClC,MAAM,EAAE,uBAAuB,EAAE,GAAG,UAAU,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACxF,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAChF,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAGxE,8EAA8E;AAC9E,cAAc;AACd,8EAA8E;AAE9E,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;AAEjD,KAAK,UAAU,GAAG;IAChB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE3B,2EAA2E;IAC3E,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC7C,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAChD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAEnD,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;IACjF,IAAI,CAAC,WAAW;QAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpF,IAAI,CAAC,YAAY;QAAE,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;IAEvF,MAAM,MAAM,GAAiB;QAC3B,MAAM;QACN,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,6BAA6B;QACtC,WAAW;QACX,YAAY;QACZ,KAAK,EAAE,OAAO;QACd,YAAY,EAAE,GAAG;QACjB,UAAU,EAAE,CAAC;QACb,gBAAgB,EAAE,MAAM;KACzB,CAAC;IAEF,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,gBAAgB,CAAC;IACpE,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC;IAE7C,OAAO,CAAC,GAAG,CAAC,gCAAgC,IAAI,OAAO,CAAC,CAAC;IACzD,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,OAAO,aAAa,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9C,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,4CAA4C,gBAAgB,GAAG,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAEhB,6EAA6E;IAC7E,MAAM,IAAI,GAAG,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,gBAAgB,EAAE,WAAW,EAAE,CAAC,CAAC;IAE5D,MAAM,OAAO,GAAgB;QAC3B,mBAAmB,EAAE,CAAC;QACtB,iBAAiB,EAAE,CAAC;QACpB,iBAAiB,EAAE,CAAC;QACpB,eAAe,EAAE,CAAC;QAClB,SAAS,EAAE,CAAC;KACb,CAAC;IAEF,IAAI,CAAC;QACH,2EAA2E;QAC3E,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAG,OAAO;YACpB,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,gBAAgB,CAAC;YAC7E,CAAC,CAAC,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;QAC9B,OAAO,CAAC,mBAAmB,IAAI,MAAM,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACnE,OAAO,CAAC,GAAG,CAAC,OAAO,MAAM,CAAC,MAAM,kBAAkB,CAAC,CAAC;QAEpD,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/E,OAAO,CAAC,mBAAmB,IAAI,MAAM,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACrE,OAAO,CAAC,GAAG,CAAC,OAAO,QAAQ,CAAC,MAAM,oBAAoB,CAAC,CAAC;QAExD,2EAA2E;QAC3E,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;QAC3D,OAAO,CAAC,iBAAiB,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACnE,OAAO,CAAC,GAAG,CAAC,OAAO,OAAO,CAAC,iBAAiB,qBAAqB,CAAC,CAAC;QAEnE,2EAA2E;QAC3E,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,MAAM,cAAc,CACzC,MAAM,EACN,OAAO,EACP,SAAS,EACT,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS,CACvC,CAAC;QACF,OAAO,CAAC,iBAAiB,IAAI,MAAM,eAAe,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QACzE,OAAO,CAAC,GAAG,CAAC,OAAO,cAAc,CAAC,MAAM,+BAA+B,CAAC,CAAC;QAEzE,2EAA2E;QAC3E,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;QAC5D,MAAM,eAAe,GAAG,MAAM,cAAc,CAC1C,MAAM,EACN,QAAQ,EACR,SAAS,EACT,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS,CACvC,CAAC;QACF,OAAO,CAAC,iBAAiB,IAAI,MAAM,eAAe,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CAAC,OAAO,eAAe,CAAC,MAAM,gCAAgC,CAAC,CAAC;QAE3E,0EAA0E;QAC1E,OAAO,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;QACzE,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC/D,MAAM,QAAQ,GAAG,IAAI,uBAAuB,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QACvE,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC9D,OAAO,CAAC,eAAe,GAAG,WAAW,CAAC,WAAW,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,OAAO,WAAW,CAAC,YAAY,iCAAiC,CAAC,CAAC;QAC9E,OAAO,CAAC,GAAG,CAAC,OAAO,WAAW,CAAC,WAAW,4BAA4B,CAAC,CAAC;QAExE,2EAA2E;QAC3E,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACzC,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAEtD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,2BAA2B,OAAO,GAAG,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,mBAAmB,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,mBAAmB,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,mBAAmB,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,mBAAmB,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;YAAS,CAAC;QACT,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAClC,OAAO,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;IACzC,OAAO,CAAC,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC"}

package/dist/census/loader.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Census DB loader — upserts GeoRecords, VariableDefinitions, and
+ * EstimateRecords into the pgsage Postgres schema.
+ *
+ * All three functions are idempotent: safe to re-run with the same data.
+ * Estimates are batched into chunks of ESTIMATE_BATCH_SIZE rows to avoid
+ * OOM on large payloads (~680K rows total for the full dataset).
+ */
+import type pg from "pg";
+import type { GeoRecord, VariableDefinition, EstimateRecord } from "./types.js";
+/**
+ * Upsert geography records into the `geography` table.
+ * Uses INSERT ... ON CONFLICT (geo_id) DO UPDATE to handle re-runs.
+ * Batches inserts to avoid exceeding PostgreSQL's parameter limit.
+ *
+ * @returns Total number of rows upserted.
+ */
+export declare function upsertGeography(pool: InstanceType<typeof pg.Pool>, records: GeoRecord[]): Promise<number>;
+/**
+ * Upsert variable definitions into the `variables` table.
+ * The full curated list is only ~247 rows so a single upsert suffices.
+ *
+ * @returns Total number of rows upserted.
+ */
+export declare function upsertVariables(pool: InstanceType<typeof pg.Pool>, definitions: VariableDefinition[]): Promise<number>;
+/**
+ * Upsert estimate records into the `estimates` table.
+ * Batches rows to avoid OOM; each batch runs in a single transaction.
+ *
+ * @returns Total number of rows upserted.
+ */
+export declare function upsertEstimates(pool: InstanceType<typeof pg.Pool>, records: EstimateRecord[]): Promise<number>;
+//# sourceMappingURL=loader.d.ts.map

package/dist/census/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/census/loader.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,EAAE,SAAS,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAahF;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,YAAY,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,EAClC,OAAO,EAAE,SAAS,EAAE,GACnB,OAAO,CAAC,MAAM,CAAC,CAsCjB;AAMD;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,YAAY,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,EAClC,WAAW,EAAE,kBAAkB,EAAE,GAChC,OAAO,CAAC,MAAM,CAAC,CAqBjB;AAMD;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,IAAI,EAAE,YAAY,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,EAClC,OAAO,EAAE,cAAc,EAAE,GACxB,OAAO,CAAC,MAAM,CAAC,CAsCjB"}

package/dist/census/loader.js

@@ -0,0 +1,127 @@
+/**
+ * Census DB loader — upserts GeoRecords, VariableDefinitions, and
+ * EstimateRecords into the pgsage Postgres schema.
+ *
+ * All three functions are idempotent: safe to re-run with the same data.
+ * Estimates are batched into chunks of ESTIMATE_BATCH_SIZE rows to avoid
+ * OOM on large payloads (~680K rows total for the full dataset).
+ */
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const ESTIMATE_BATCH_SIZE = 500;
+const GEO_BATCH_SIZE = 500;
+// ---------------------------------------------------------------------------
+// Geography
+// ---------------------------------------------------------------------------
+/**
+ * Upsert geography records into the `geography` table.
+ * Uses INSERT ... ON CONFLICT (geo_id) DO UPDATE to handle re-runs.
+ * Batches inserts to avoid exceeding PostgreSQL's parameter limit.
+ *
+ * @returns Total number of rows upserted.
+ */
+export async function upsertGeography(pool, records) {
+    if (records.length === 0)
+        return 0;
+    let total = 0;
+    const batches = chunkArray(records, GEO_BATCH_SIZE);
+    for (const batch of batches) {
+        // Build multi-row VALUES clause
+        const values = [];
+        const placeholders = batch.map((rec, i) => {
+            const base = i * 6;
+            values.push(rec.geoId, rec.stateFips, rec.countyFips ?? null, rec.stateName, rec.countyName ?? null, rec.level);
+            return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6})`;
+        });
+        await pool.query(`INSERT INTO geography (geo_id, state_fips, county_fips, state_name, county_name, level)
+       VALUES ${placeholders.join(", ")}
+       ON CONFLICT (geo_id) DO UPDATE SET
+         state_fips  = EXCLUDED.state_fips,
+         county_fips = EXCLUDED.county_fips,
+         state_name  = EXCLUDED.state_name,
+         county_name = EXCLUDED.county_name,
+         level       = EXCLUDED.level`, values);
+        total += batch.length;
+    }
+    return total;
+}
+// ---------------------------------------------------------------------------
+// Variables
+// ---------------------------------------------------------------------------
+/**
+ * Upsert variable definitions into the `variables` table.
+ * The full curated list is only ~247 rows so a single upsert suffices.
+ *
+ * @returns Total number of rows upserted.
+ */
+export async function upsertVariables(pool, definitions) {
+    if (definitions.length === 0)
+        return 0;
+    const values = [];
+    const placeholders = definitions.map((def, i) => {
+        const base = i * 4;
+        values.push(def.code, def.label, def.concept, def.category);
+        return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4})`;
+    });
+    await pool.query(`INSERT INTO variables (variable_code, label, concept, category)
+     VALUES ${placeholders.join(", ")}
+     ON CONFLICT (variable_code) DO UPDATE SET
+       label    = EXCLUDED.label,
+       concept  = EXCLUDED.concept,
+       category = EXCLUDED.category`, values);
+    return definitions.length;
+}
+// ---------------------------------------------------------------------------
+// Estimates
+// ---------------------------------------------------------------------------
+/**
+ * Upsert estimate records into the `estimates` table.
+ * Batches rows to avoid OOM; each batch runs in a single transaction.
+ *
+ * @returns Total number of rows upserted.
+ */
+export async function upsertEstimates(pool, records) {
+    if (records.length === 0)
+        return 0;
+    let total = 0;
+    const batches = chunkArray(records, ESTIMATE_BATCH_SIZE);
+    for (const batch of batches) {
+        const client = await pool.connect();
+        try {
+            await client.query("BEGIN");
+            const values = [];
+            const placeholders = batch.map((rec, i) => {
+                const base = i * 4;
+                values.push(rec.geoId, rec.variableCode, rec.estimate ?? null, rec.marginError ?? null);
+                return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4})`;
+            });
+            await client.query(`INSERT INTO estimates (geo_id, variable_code, estimate, margin_error)
+         VALUES ${placeholders.join(", ")}
+         ON CONFLICT (geo_id, variable_code) DO UPDATE SET
+           estimate     = EXCLUDED.estimate,
+           margin_error = EXCLUDED.margin_error`, values);
+            await client.query("COMMIT");
+            total += batch.length;
+        }
+        catch (err) {
+            await client.query("ROLLBACK");
+            throw err;
+        }
+        finally {
+            client.release();
+        }
+    }
+    return total;
+}
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+function chunkArray(arr, size) {
+    const result = [];
+    for (let i = 0; i < arr.length; i += size) {
+        result.push(arr.slice(i, i + size));
+    }
+    return result;
+}
+//# sourceMappingURL=loader.js.map