@nitpicker/archive 0.0.0 → 0.25.0

package/LICENSE

@@ -0,0 +1,191 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to the Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by the Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding any notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2025 D-ZERO Co., Ltd.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/README.md

@@ -0,0 +1,106 @@
+# @nitpicker/archive
+
+`.nitpicker` アーカイブ（tar + SQLite）の読み書きを担う内部パッケージです。スキーマ定義・マイグレーション・`Archive` / `ArchiveAccessor`・tar 展開キャッシュ・concat / split・body hash・URL alias・技術スタック検出のロールアップ・テンプレート分類・保存済み HTML スナップショットのセレクタ照合と文字列・正規表現検索（`html-snapshot/`。ネイティブの `@nitpicker/core` を呼ぶ薄いラッパーと、その型）・スコープ判定・エラー分類を含みます。
+
+ヘッドレスブラウザ（puppeteer / `@d-zero/beholder`）や `@d-zero/dealer` には依存しないため、読み取り専用の経路（[@nitpicker/query](../query/README.md)、viewer、mcp-server、report-\*）はクローラーを引き込まずにアーカイブを扱えます。クロール時の書き込みは [@nitpicker/crawler](../crawler/README.md) がこのパッケージ経由で行います。
+
+## 使い方
+
+公開 API は機能単位のサブパスで export しています（パッケージルート `.` の export はありません）。
+
+```ts
+import Archive from '@nitpicker/archive/archive';
+import { classifyErrorKind } from '@nitpicker/archive/error-kind/classify-error-kind';
+
+// `await using` が SQLite ハンドルを確実に閉じる
+await using accessor = await Archive.openCached('/path/to/site.nitpicker');
+const config = await accessor.getConfig();
+
+console.log(classifyErrorKind('net::ERR_NAME_NOT_RESOLVED')); // 'dns'
+```
+
+利用可能なサブパスは `package.json` の `exports` を参照してください。
+
+## テンプレート分類（page-cluster）
+
+テンプレート分類（`src/template-classification/`）は、ページを DOM 構造の類似性でクラスタリングします。計算はすべて [@nitpicker/core](../core/README.md) のネイティブアドオン（Rust の `nitpicker_template_cluster`）が実行し、`src/template-classification/page-cluster/` にはそれを駆動するドライバ（`resolve-page-cluster-keys.ts` の `resolvePageClusterKeys`）だけがあります。ドライバはページのシグナル（URL パス・スタイルシート・ホスト）を渡してから、各ページの HTML を必要な読み込みでだけ取り出して渡し、ページごとのキーとクラスタごとの reason を受け取ります。ドライバはこのディレクトリの外（`@nitpicker/core` を除く）に依存せず、パッケージの `exports` にも載せていません。呼び出しは `classify-page-templates.ts` / `create-page-cluster-factory.ts` に閉じています（ほかはテスト専用の `golden/`）。分類器の `clusterKey` は nitpicker 側で `templateKey` と呼び替えます。
+
+出力は既存アーカイブに保存済みの `templateKey` と reason にバイト単位で一致し、`src/template-classification/golden/` の golden がそれを固定しています。
+
+### アルゴリズム概観
+
+`clusterKey` がどう決まるかを知っておくと、分類結果の解釈（なぜこの 2 ページが同じ `templateKey` なのか）がしやすくなります。実装詳細の WHY は `nitpicker_template_cluster` の各モジュールの doc comment が正です。
+
+#### 全体パイプライン
+
+```mermaid
+flowchart TD
+    IN[入力ページ集合] --> P0["Pass 0: ブロッキング<br>URL パス + first-party CSS 集合 → blockKey<br>（orphan ページの再割当を含む）"]
+    P0 --> GATE{"ブロックごと:<br>類似度表 n(n−1)/2×8 バイト ≤ メモリ上限?<br>かつ 全ページ計算の合計 ≤ 20,000 ページ?"}
+
+    GATE -- "yes（全ページ）" --> CHROME_ALL["chrome discovery（全ページをクラスタリングするブロックのページ全体）<br>ランドマーク署名の度数分布に auto-cut<br>→ グローバル chrome 除外 / ローカル chrome 再注入"]
+    CHROME_ALL --> SA_ALL["Stage A × そのブロック<br>深さキャップ → tokenize → 固有クラス除去 →<br>complete-linkage + auto-cut → 包含割当"]
+
+    GATE -- "no（サンプル）" --> RES["リザーバサンプリング<br>（最大 100 ページ、決定的シード）"]
+    RES --> SA_SAMPLE["chrome discovery + Stage A<br>（サンプルのみ、ブロックの最後のページを読んだ時点で flush）"]
+    SA_SAMPLE --> P1B["Pass 1b: 非サンプルページを<br>max-Jaccard で最寄りクラスタへ割当"]
+
+    SA_ALL --> SB["Stage B: ブロック越えマージ<br>（不動点ループ、下図）"]
+    P1B --> SB
+    SB --> OUT["clusterKey を入力順に出力"]
+```
+
+- **Pass 0（ブロッキング）** — HTML を読まず、URL パスと first-party stylesheet 集合だけで粗く分割します。高価な構造比較を同一ブロック内に閉じ込め、コーパス全体の比較コストを O(n²) から劇的に減らします。stylesheet を持たない orphan ページは同一セクションの CSS ブロックへ再割当されます
+- **chrome discovery** — 全ページのランドマーク署名の度数分布に auto-cut を当て、閾値以上を「グローバル chrome」（サイト共通のヘッダー等）として比較から除外し、閾値未満かつ 2 ページ以上に出現するものを「ローカル chrome」（セクション固有のナビ等）としてトークン再注入します
+- **Stage A（ブロック内クラスタリング）** — ブロックごとに直線的に処理します。本文ルート（`contentRoot` → `<main>` → 内蔵リスト）の深さキャップ（候補深度 2〜10 を全走査して knee を探す自動選択）→ tokenize → **ページ固有クラスの除去**（ブロック内で 1 ページにしか現れないクラスは `<article class="outline">` のようなページ識別ラベルであってテンプレート構造ではないので、`allowedClasses` で除いて再 tokenize します。10 ページ以上のブロックのみ）→ complete-linkage 階層クラスタリング → max-gap auto-cut でカット高を決定 → 最後に包含関係にあるクラスタを吸収する包含割当（割当チェーンを辿り、循環はメンバー最大のクラスタをルートに選んで解決）
+- **Pass 1b（サンプルに落としたブロックのみ）** — Stage A の類似度表がメモリ上限（`--template-memory-budget <MiB>` または `NITPICKER_TEMPLATE_MEMORY_BUDGET`、既定 512 MiB、0 なら全ブロック）を超えるブロックだけ、リザーバサンプリング（最大 100 ページ、ブロックキーをシードにした決定的乱数）で代表させ、サンプル外のページは Stage A 完了後に max-Jaccard で最寄りクラスタへ一括割当する（tokenize にはサンプルで学習した許可クラス集合を適用し、サンプル側と同じ規則で比較します）。そのブロックのメモリ使用量はブロックのページ数ではなくサンプルサイズに比例します。もう 1 つの上限は、全ページで計算するブロックのページ（HTML の解析結果を計算が終わるまで全ブロック分抱える。1 ページ約 60KB）の合計で、`@nitpicker/core` の `TEMPLATE_EXACT_PAGE_LIMIT`（20,000 ページ、固定）です。超えるときは、標本より大きい（100 ページを超える）ブロックを、ページ数が多い順（同数ならブロックの出現順）に、収まるまでサンプルに落とします。100 ページ以下のブロックはサンプルにしても保持ページが減らないので落とさず、それだけで上限を超えるときは超えたまま保持します。ブロックの判定はページ数だけで決まり（重複する集合のまとめ込みはしません）、判定したブロックは `block-sampled` の進捗イベントと分類結果の `sampledBlocks` に、理由（`table-over-budget`、`pages-over-limit`）つきで残ります。サンプルに落としたブロックを含むマージ済みグループは Stage B でも 100 メンバーに抑えます
+- **Stage B（ブロック越えマージ）** — ブロック分割はあくまで比較コスト削減のためなので、最後に同一テンプレートがブロックを跨いで分かれていないか再統合します。これが唯一の反復処理です（次節）。収束後、確定した最終クラスタごとに reason（`TemplateClusterReason`）を 1 つずつ組み立てます — 追加の全コーパススキャンではなく、Stage A/B が既に計算済みの中間データ（quorum core、landmark インスタンス、ブロッキング根拠）を再利用するだけなので、クラスタ数にしか比例しません
+
+#### Stage B: ブロック越え統合の不動点ループ
+
+```mermaid
+flowchart TD
+    START["ラウンド開始（最大 10 ラウンド）"] --> CORE["現在のプール済みメンバーから再計算:<br>文書頻度 → distinctive tokens → quorum core（80%）"]
+    CORE --> FINE["fine stage（単一 union-find 上で 3 経路）:<br>① complete-linkage（固定 0.8）<br>② 包含割当（0.9、チェーン走査 + サイクル解決）<br>③ shape-Jaccard（0.9、複数ページユニットのみ）"]
+    FINE --> Q1{"fine でマージ発生?"}
+    Q1 -- yes --> APPLY1["マージ適用（メンバー統合）"]
+    APPLY1 --> START
+    Q1 -- no --> L2["L2 stage:<br>L2 signature 包含 + shell 相互裏付け<br>（shell は auto-cut で自己発見）"]
+    L2 --> Q2{"L2 でマージ発生?"}
+    Q2 -- yes --> APPLY2["マージ適用"]
+    APPLY2 --> START
+    Q2 -- no --> DONE["収束 — 全ユニットのキーが不動点に到達"]
+```
+
+マージが起きるとユニットのメンバー構成が変わり、文書頻度も quorum core も変わります。そのため毎ラウンド、統合後のプールから全指標を**再計算**してマージを再試行します。fine stage・L2 stage の両方でマージが 1 件も出なくなった時点で不動点に到達したとみなして収束します（安全弁として最大 10 ラウンド。実データでは 7 ラウンド以内に収束）。L2 stage は fine stage が空振りしたラウンドでしか実行されない最後の粗い経路で、誤マージ防止のために shell（ランドマーク由来トークン）の相互裏付けを要求します。
+
+fine stage・L2 stage いずれの経路で提案されたマージも、適用前に**凝集度ガード**を通ります: 統合後の quorum core が統合前の core に対して一定比率を下回るなら、そのマージは破棄されます。個々の経路が「統合前のペア類似度」だけを見て提案する一方、複数ラウンドにわたる連鎖的な統合は「統合後に実際どれだけまとまっているか」を悪化させ得る（無関係なテンプレート同士が少しずつ吸収し合う catch-all 化）ため、経路をまたいだ単一のチェックポイントとして機能します。あわせて、L2 stage 自体もラウンド開始前に判別力を検査し、参加ユニット全体が同一の署名形状に潰れている（`main` 直下の浅い階層しか手がかりが残っていない等）場合は、そのラウンドの L2 比較を丸ごとスキップします。詳細は `merge_cross_block_clusters.rs` / `filter_merges_by_cohesion.rs` / `l2_signature.rs` の doc comment を参照してください。
+
+凝集度ガードの前に、Stage A が同一ブロック内で別クラスタと判定した（双方 3 ページ以上の）ユニット同士を、構造トークンに実質的な差がある限り統合しない**分離尊重ガード**も通ります。生の core では閾値に届かないが、クラス名を除くと骨格が一致する組（BEM 名だけが違う同一骨格）は対象外です。詳細は `filter_merges_by_stage_a_separation.rs` の doc comment を参照してください。
+
+#### Self-tuning
+
+閾値の多くは **max-gap auto-cut**（度数分布の隣接ギャップ最大の中点を境界とする）でデータから自己発見されます。① Stage A のカット高、② Stage B の shell 判定、③ chrome discovery のグローバル/ローカル判定、の 3 箇所で同一プリミティブを再利用しているので、サイトごとにハイパーパラメータをチューニングする必要はありません。詳細は `auto_cut_threshold.rs` の doc comment を参照してください。例外的に Stage B fine stage の complete-linkage だけは固定閾値 0.8 を使います（理由は `merge_cross_block_clusters.rs` を参照）。
+
+### golden（出力を固定するテスト）
+
+`src/template-classification/golden/` は、分類器の出力（ページごとの `templateKey` と、`page_template_clusters` に保存される reason JSON の文字列）を `__golden__/` に固定し、全件一致を確かめます。入力は、決定的に生成する合成サイトと `page-cluster/__fixtures__/` の HTML で、ドライバに直接渡します。合成サイトは、小さい 1 規模と、約 2 万ページ強の 1 規模です。後者はメモリ上限を 3 通りに変えて記録します。既定のメモリ上限では最大のブロックだけが保持ページ数の上限でサンプルに落ち、メモリ上限 0 では全ブロックが落ち、メモリ上限を絞った場合は一部のブロックだけが落ちます。メモリ上限 0 の記録は、全ブロックをサンプルから計算した結果で、この方式の出力を固定します。分類器を書き換えても出力が同じであることのゲートです。アーカイブ側のアダプタ（`createPageClusterFactory` など）を含む本番経路全体は、実アーカイブで `scripts/template-classification-golden.mjs` を使って確かめます。
+
+出力を意図して変えるときだけ、次のコマンドで `__golden__/` を更新し、理由を PR に書きます（パスを付けないと他の spec の snapshot まで更新されます）。
+
+```sh
+yarn vitest run packages/@nitpicker/archive/src/template-classification/golden --exclude '.claude/**' -u
+```
+
+差分を読むときは `inputDigest` を先に見ます。`inputDigest` が変わっていれば入力（生成器か fixture）が変わっています。`inputDigest` が同じなのにほかのフィールドが変わっていれば、分類器の出力が変わっています。
+
+`page-cluster/__fixtures__/` の HTML は、`@nitpicker/core` の Rust テスト（htmlparser2 との突き合わせ）も読みます。fixture を足したり変えたりしたら、core の `scripts/record-htmlparser2-events.mjs` で記録済みイベントを取り直します（手順はスクリプト冒頭）。取り直すまで core のテストが落ちます。
+
+## 関連リンク
+
+- [Nitpicker README](../../../README.md)
+- [ARCHITECTURE.md](../../../ARCHITECTURE.md)
+
+## ライセンス
+
+Apache-2.0

package/lib/apply-connection-pragmas.d.ts

@@ -0,0 +1,18 @@
+import type { Knex } from 'knex';
+/**
+ * Applies the connection-level PRAGMAs that govern foreign-key enforcement
+ * and BLOB-read performance. These are **per-connection** settings (libsql
+ * resets them when a new connection is opened), so they must be reapplied
+ * every time `Database.connect` runs — not just on first-time schema
+ * initialization. Keeping them separate from `initSchema`'s one-shot path
+ * also lets `page_size` (which only takes effect against an empty DB)
+ * stay gated behind the existence check.
+ *
+ * The `cache_size` and `mmap_size` values are tuned for reads. The viewer
+ * read model build swaps the two for its one long bulk write through
+ * `withBulkWritePragmas`, which puts back the values it found afterwards.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @example
+ * await applyConnectionPragmas(knex); // right after the connection opens
+ */
+export declare function applyConnectionPragmas(instance: Knex): Promise<void>;

package/lib/apply-connection-pragmas.js

@@ -0,0 +1,40 @@
+/**
+ * Applies the connection-level PRAGMAs that govern foreign-key enforcement
+ * and BLOB-read performance. These are **per-connection** settings (libsql
+ * resets them when a new connection is opened), so they must be reapplied
+ * every time `Database.connect` runs — not just on first-time schema
+ * initialization. Keeping them separate from `initSchema`'s one-shot path
+ * also lets `page_size` (which only takes effect against an empty DB)
+ * stay gated behind the existence check.
+ *
+ * The `cache_size` and `mmap_size` values are tuned for reads. The viewer
+ * read model build swaps the two for its one long bulk write through
+ * `withBulkWritePragmas`, which puts back the values it found afterwards.
+ * @param instance - The Knex query builder instance connected to the database.
+ * @example
+ * await applyConnectionPragmas(knex); // right after the connection opens
+ */
+export async function applyConnectionPragmas(instance) {
+    // Foreign-key enforcement defaults to OFF on every new SQLite
+    // connection. Required for ON DELETE CASCADE on `page_html_ref`,
+    // `technology_signals`, `page_technologies`, `page_jsonld`, and the
+    // `page_main_content_*` tables to fire.
+    await instance.raw('PRAGMA foreign_keys = ON');
+    await instance.raw('PRAGMA wal_autocheckpoint = 1000');
+    // Negative value = KiB of memory (64 MiB). Helps large BLOB scans.
+    //
+    // Empirically validated against larger values on a 10 GB archive:
+    // bumping to 512 MiB regressed `getSummary` (1.9s → 5.7s), `pages`
+    // (2.3s → 21s), and `images` (3.7s → 12s) — libsql's page eviction
+    // policy interacts poorly with a cache sized comparable to the
+    // host's page-cache window when the DB itself far exceeds RAM.
+    // 64 MiB stays the sweet spot.
+    await instance.raw('PRAGMA cache_size = -65536');
+    // 256 MiB mmap window. SQLite falls back to read() past this so the
+    // limit is a soft ceiling, not a hard one. A 4 GiB window was
+    // catastrophic on a 10 GB archive on macOS (summary 1.9s → 43s,
+    // pages 2.3s → 21s) — the kernel's read-ahead policy and libsql's
+    // mmap path interact badly when the window can cover most of the
+    // DB. Keep this conservative.
+    await instance.raw('PRAGMA mmap_size = 268435456');
+}

package/lib/archive-accessor.d.ts

@@ -0,0 +1,265 @@
+import type { Database } from './database.js';
+import type { Config, DatabaseEvent, PageFilter } from './types.js';
+import type { ParseURLOptions } from '@d-zero/shared/parse-url';
+import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
+import Page from './page.js';
+import Resource from './resource.js';
+export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
+    #private;
+    /**
+     * Whether this accessor was opened in read-only mode (no filesystem
+     * mutation on `tmpDir`).
+     */
+    get readOnly(): boolean;
+    /**
+     * The absolute path to the temporary working directory used by this accessor.
+     */
+    get tmpDir(): string;
+    /**
+     * Creates a new ArchiveAccessor instance.
+     * @param tmpDir - The path to the temporary directory containing the archive data.
+     * @param db - The Database instance for querying the SQLite database.
+     * @param options - Construction options.
+     * @param options.readOnly - When `true`, helpers must not mutate the
+     *   filesystem under `tmpDir` (used for live-crawl / stub-mode opens
+     *   where any write would race the crawler).
+     */
+    constructor(tmpDir: string, db: Database, options?: {
+        readOnly?: boolean;
+    });
+    /**
+     * Enables `await using accessor = ...`. Delegates to {@link close} with
+     * the default timeout — callers that need a non-default `timeoutMs`
+     * must call `close` explicitly instead of relying on disposal.
+     *
+     * Dispatches through the instance's own `close`, so an `Archive`
+     * (which overrides `close`) gets its full teardown here too — this
+     * method does not need to be re-implemented on subclasses.
+     */
+    [Symbol.asyncDispose](): Promise<void>;
+    /**
+     * Closes the underlying database connection.
+     *
+     * This is the **read-only** close path: it releases the SQLite handle and
+     * does nothing else. The temporary working directory is left untouched and
+     * no `.nitpicker` archive is produced. This makes it safe to call from
+     * read-only consumers (e.g. the viewer attached to an in-progress crawl's
+     * tmpDir), where touching the filesystem would race with — or destroy —
+     * the live crawler's working state.
+     *
+     * Subclasses that own the archive's lifecycle (notably `Archive`)
+     * override this to add write/cleanup steps.
+     *
+     * **Idempotent and concurrent-safe**: the first invocation captures the
+     * close promise; later invocations (from the same caller, a shutdown
+     * signal handler, or a parallel manager teardown) await the same
+     * promise and resolve together. If `db.destroy()` rejects, the
+     * rejection propagates to *all* awaiters and the accessor stays
+     * latched closed — a hung knex pool is not safe to "retry close".
+     *
+     * **Bounded**: when the optional `timeoutMs` (default {@link
+     * DEFAULT_CLOSE_TIMEOUT_MS}) elapses before `db.destroy()` settles, the
+     * call resolves with a warning. This prevents a viewer shutdown from
+     * being held for the underlying pool's 10-minute `acquireTimeoutMillis`
+     * when the live crawler holds the SQLite write lock.
+     * @param options - Close options.
+     * @param options.timeoutMs - Milliseconds to wait for `db.destroy()`
+     *   before giving up. Use `Infinity` to wait indefinitely (only
+     *   advisable in tests and batch jobs that own the DB exclusively).
+     */
+    close(options?: {
+        timeoutMs?: number;
+    }): Promise<void>;
+    /**
+     * Retrieves anchor (link) data for a specific page by its database ID.
+     * @param pageId - The database ID of the page whose anchors to retrieve.
+     * @returns An array of anchor records found on the page.
+     */
+    getAnchorsOnPage(pageId: number): Promise<any[]>;
+    /**
+     * Retrieves the audios within the given page's detected main content
+     * region, from `page_main_content_audios`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered audio rows.
+     */
+    getAudiosOfPage(pageId: number): Promise<import("./meta/types.js").MainContentAudioRow[]>;
+    /**
+     * Retrieves the button-like elements within the given page's detected
+     * main content region, from `page_main_content_buttons`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered button rows.
+     */
+    getButtonsOfPage(pageId: number): Promise<import("./meta/types.js").MainContentButtonRow[]>;
+    /**
+     * Retrieves the canvases within the given page's detected main content
+     * region, from `page_main_content_canvases`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered canvas rows.
+     */
+    getCanvasesOfPage(pageId: number): Promise<import("./meta/types.js").MainContentCanvasRow[]>;
+    /**
+     * Retrieves the crawl configuration stored in the archive database.
+     * @returns The parsed {@link Config} object.
+     */
+    getConfig(): Promise<Config>;
+    /**
+     * Retrieves the Web Components (custom elements) within the given page's
+     * detected main content region, from `page_main_content_custom_elements`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered custom-element rows.
+     */
+    getCustomElementsOfPage(pageId: number): Promise<import("./meta/types.js").MainContentCustomElementRow[]>;
+    /**
+     * Reads a data file stored at the archive root (for example an inventory source list) by name.
+     * @param name - The base name of the data file (without extension).
+     * @param format - The file format: `'json'` (default), `'txt'`, or `'html'`.
+     * @returns The parsed JSON object for `'json'` format, or a string for `'txt'`/`'html'` format.
+     */
+    getData<T>(name: string, format?: 'json'): Promise<T>;
+    /**
+     * Reads custom data stored in the archive by name as a string.
+     * @param name - The base name of the data file (without extension).
+     * @param format - The file format: `'txt'` or `'html'`.
+     * @returns The file contents as a string.
+     */
+    getData(name: string, format?: 'txt' | 'html'): Promise<string>;
+    /**
+     * Retrieves the headings within the given page's detected main content
+     * region, from `page_main_content_headings`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered heading rows.
+     */
+    getHeadingsOfPage(pageId: number): Promise<import("./meta/types.js").MainContentHeadingRow[]>;
+    /**
+     * Reads the HTML snapshot of a page from the archive.
+     *
+     * HTML is stored as zstd-compressed BLOBs in `page_html_blobs` (keyed by
+     * SHA-256 of the raw bytes) with `page_html_ref` linking `page_id → hash`.
+     * The read is a straight join + decompress; writer- and read-only (stub)
+     * accessors take the same code path because nothing here touches the
+     * filesystem.
+     *
+     * Returns `null` when the page row exists but has no stored body — for
+     * example a redirect source, a non-HTML resource (PDF), a page that
+     * failed to render, or an external page (whose row is metadata-only).
+     * Distinguishing "no body stored" from "empty body" is preserved: an
+     * empty HTML string returns `''`, not `null`.
+     *
+     * Throws if the page's referenced blob is missing, its codec marker is
+     * unrecognised, its zstd data is corrupt, or it decompresses past
+     * `MAX_DECODED_SNAPSHOT_BYTES` — each indicates an archive that was
+     * truncated, damaged or written by a future tool, none of which we
+     * silently paper over. A decode failure names the page and the blob hash
+     * (`HTML snapshot of page <id> (page_html_blobs hash <hex>) could not be decoded: …`).
+     * @param pageId - The database id of the page to read.
+     * @returns The HTML content as a UTF-8 string, or `null` when no body
+     *   is stored for `pageId`.
+     * @example
+     * const html = await accessor.getHtmlOfPage(pageId);
+     * if (html === null) {
+     *   // page has no stored body — redirect source / non-HTML / failed render
+     * } else {
+     *   processHtml(html);
+     * }
+     */
+    getHtmlOfPage(pageId: number): Promise<string | null>;
+    /**
+     * Retrieves the iframes within the given page's detected main content
+     * region, from `page_main_content_iframes`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered iframe rows.
+     */
+    getIframesOfPage(pageId: number): Promise<import("./meta/types.js").MainContentIframeRow[]>;
+    /**
+     * Retrieves the JSON-LD / SpeculationRules entries for the given page,
+     * parsed back from the `page_jsonld` table.
+     * @param pageId - The database id of the page.
+     * @returns Ordered entries with `kind`, `type`, `raw`, `parsed`, `parseError`.
+     */
+    getJsonLdOfPage(pageId: number): Promise<import("./meta/types.js").JsonLdRow[]>;
+    /**
+     * Returns the underlying Knex query builder instance for direct SQL access.
+     * Enables advanced queries (GROUP BY, HAVING, JOINs) at the database layer
+     * for performance-critical operations on large datasets.
+     * @returns The Knex instance connected to the SQLite database.
+     */
+    getKnex(): import("knex").Knex<any, any[]>;
+    /**
+     * Retrieves the images within the given page's detected main content
+     * region, from `page_main_content_images`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered image rows.
+     */
+    getMainContentImagesOfPage(pageId: number): Promise<import("./meta/types.js").MainContentImageRow[]>;
+    /**
+     * Retrieves the tables within the given page's detected main content
+     * region, from `page_main_content_tables`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered table rows.
+     */
+    getMainContentTablesOfPage(pageId: number): Promise<import("./meta/types.js").MainContentTableRow[]>;
+    /**
+     * Retrieves all pages from the archive, optionally filtered by type.
+     * Eagerly loads redirect relationships (`redirectFrom`) but does NOT load
+     * anchor or referrer relationships.
+     * Use {@link getPagesWithRefs} if you need those relationships.
+     * @param filter - An optional filter to narrow the results (e.g., `'internal-page'`, `'external-page'`).
+     * @returns An array of {@link Page} instances.
+     */
+    getPages(filter?: PageFilter): Promise<Page[]>;
+    /**
+     * Retrieves pages with their related data (redirects, anchors, referrers) in batches.
+     * Processes pages in chunks of `limit` size, calling the callback for each batch.
+     * @param limit - The maximum number of pages to load per batch.
+     * @param callback - A function called for each batch of pages with the current offset and total count.
+     * @param options - Optional URL parsing options and whether to include referrer relationships.
+     */
+    getPagesWithRefs(limit: number, callback: (pages: Page[], currentOffset: number, max: number) => void | Promise<void>, options?: ParseURLOptions & {
+        withRefs?: boolean;
+    }): Promise<void>;
+    /**
+     * Retrieves the confidence-combined technology roll-up for the given
+     * page, from the `page_technologies` table.
+     * @param pageId - The database id of the page.
+     * @returns Technology rows with category, version, confidence, signalCount.
+     */
+    getPageTechnologiesOfPage(pageId: number): Promise<import("./meta/types.js").PageTechnologyRow[]>;
+    /**
+     * Retrieves pages that link to the specified page (incoming links).
+     * @param pageId - The database ID of the target page.
+     * @returns An array of referrer records.
+     */
+    getReferrersOfPage(pageId: number): Promise<any[]>;
+    /**
+     * Retrieves page URLs that reference the specified resource.
+     * @param pageId - The database ID of the resource.
+     * @returns An array of page URL strings that reference this resource.
+     */
+    getReferrersOfResource(pageId: number): Promise<string[]>;
+    /**
+     * Retrieves all sub-resources (CSS, JS, images, etc.) stored in the archive.
+     * @returns An array of {@link Resource} instances.
+     */
+    getResources(): Promise<Resource[]>;
+    /**
+     * Retrieves a flat list of all resource URLs stored in the archive.
+     * @param onProgress - Forwarded to {@link Database.getResourceUrlList} —
+     *   see that method's docs.
+     * @returns An array of resource URL strings.
+     */
+    getResourceUrlList(onProgress?: (scannedUpToId: number, maxId: number) => void): Promise<string[]>;
+    /**
+     * Retrieves the raw technology-detection signals for the given page,
+     * from the `technology_signals` table.
+     * @param pageId - The database id of the page.
+     * @returns Ordered signal rows with technology, signalType, evidence, weight.
+     */
+    getTechnologySignalsOfPage(pageId: number): Promise<import("./meta/types.js").TechnologySignalRow[]>;
+    /**
+     * Retrieves the videos within the given page's detected main content
+     * region, from `page_main_content_videos`.
+     * @param pageId - The database id of the page.
+     * @returns Ordered video rows.
+     */
+    getVideosOfPage(pageId: number): Promise<import("./meta/types.js").MainContentVideoRow[]>;
+}