@nitpicker/crawler 0.12.0 → 0.13.0

package/lib/archive/init-schema.d.ts

@@ -13,47 +13,43 @@ export declare function applyConnectionPragmas(instance: Knex): Promise<void>;
 /**
  * Initializes the archive database schema if tables do not exist.
  *
- * Schema notes:
+ * The schema is composed of four groups, each owned by a dedicated DDL
+ * function so the migration script (`scripts/migrate-to-0.13.mjs`) can
+ * provision the exact same shapes on archives it upgrades:
  *
- * - **Meta columns (v2)**: pages carries ~47 flat columns derived from
- *   beholder 3.0.0's nested Meta shape (`canonical`, `og_*`, `twitter_*`,
- *   `robots_*`, document basics, editorial fields) plus a `meta_extras`
- *   JSON column for everything not flattened. URL-shaped columns are
- *   absolutised against the page URL before write (see
- *   `archive/meta/derive-flat-from-meta.ts`).
- * - **Denormalised aggregates** (`tag_count`, `jsonld_count`,
- *   `tags_providers_csv`): computed at write time from `meta.tags` /
- *   `meta.jsonLd` to avoid N+1 GROUP BY at Sheets-render / page-detail time.
- *   Plan: "ファイルサイズが多少増えてもいいから取り出しパフォーマンスを優先".
- * - **Per-page timestamps** (`firstCrawledAt`, `lastCrawledAt`): UNIX ms.
- *   Written by `#insertPage` on INSERT (`first = last = now`) and UPDATE
- *   (`last = now`, `first` preserved). `resetFailedPages` deliberately
- *   leaves them alone so failure-reset does not erase the last-success
- *   record.
- * - **`page_tags`** (Wappalyzer): per-provider × external-id row shape, plus
- *   `categories`/`sources` JSON columns. Compound indexes
- *   `(provider, externalId)` / `(provider, pageId)` are pre-built for the
- *   Phase 2+ "find duplicate IDs across pages" and "list pages using
- *   provider X" hot paths — Phase 1 read perf > storage cost trade-off.
- * - **`page_jsonld`** (JSON-LD / SpeculationRules): one row per
- *   `<script type="application/ld+json">` or `<script type="speculationrules">`.
- *   `raw` is stored uncompressed (SQLite overflow pages handle large rows);
- *   if cross-archive bulk export becomes a use case, add a `codec` column
- *   à la `page_html_blobs`. Compound `(type, pageId)` accelerates streaming
- *   `list_pages_by_jsonld_type` JOINs.
- * - **HTML snapshots** (`page_html_blobs` + `page_html_ref`): unchanged
- *   from v1. zstd-compressed BLOBs keyed by SHA-256 for content-addressable
- *   dedup. WITHOUT ROWID via raw SQL because knex's schema builder cannot
- *   express it.
- * - **PRAGMA `page_size` and `journal_mode`** are set BEFORE any
- *   `CREATE TABLE` because SQLite only honors `page_size` changes against
- *   an empty database, and `journal_mode = WAL` is persistent. Other
- *   per-connection PRAGMAs live in {@link applyConnectionPragmas}.
+ * - **`info`** (inline below): single-row crawl configuration.
+ * - **Ref / header dictionary tables** ({@link createRefTables}):
+ *   `url_refs`, `content_type_refs`, `text_refs`, `json_refs`,
+ *   `blob_refs`, `header_name_refs`, `header_value_refs`, `header_sets`,
+ *   `header_set_entries`, `header_flags`. See that file for column-level
+ *   rationale and the reason `blob_refs` uses a regular rowid PK instead
+ *   of WITHOUT ROWID.
+ * - **Entity / edge tables** ({@link createEntityTables}):
+ *   `content_items`, `page_meta`, `resource_items`, `anchor_edges`,
+ *   `resource_ref_edges`, `image_items` — the write-model the crawler
+ *   writes during a crawl and every reader queries. Must run AFTER
+ *   `createRefTables` because most entity tables reference ref-table PKs.
+ * - **Adjunct tables** ({@link createAdjunctTables}): `page_errors`,
+ *   `crawl_errors`, `page_tags`, `page_jsonld`, `inventory_runs`,
+ *   `analysis_text_refs` + `analysis_violations`, `page_html_blobs` +
+ *   `page_html_ref`. Must run AFTER `createEntityTables` because the
+ *   page-scoped tables FK into `content_items(id)`.
  *
- * Pre-0.10 → 0.10 migration is intentionally absent. `assertCompatibleVersion`
- * (called before `initSchema`) rejects pre-0.10 archives with a friendly
- * error pointing the operator at `scripts/migrate-to-0.10.mjs`; `v0.x`
- * policy allows breaking changes.
+ * The legacy flat write-model tables (`pages` / `anchors` / `images` /
+ * `resources` / `resources-referrers`) are deliberately NOT created:
+ * they only exist inside pre-0.13 archives, where they serve as the
+ * populate source for `scripts/migrate-to-0.13.mjs` before that script
+ * drops them.
+ *
+ * **PRAGMA `page_size` and `journal_mode`** are set BEFORE any
+ * `CREATE TABLE` because SQLite only honors `page_size` changes against
+ * an empty database, and `journal_mode = WAL` is persistent. Other
+ * per-connection PRAGMAs live in {@link applyConnectionPragmas}.
+ *
+ * Pre-0.13 migration is intentionally absent. `assertCompatibleVersion`
+ * (called before `initSchema`) rejects older archives with a friendly
+ * error naming the migration scripts to run; `v0.x` policy allows
+ * breaking changes.
  * @param instance - The Knex query builder instance connected to the database.
  */
 export declare function initSchema(instance: Knex): Promise<void>;