@nitpicker/crawler 0.17.0 → 0.18.1

package/lib/archive/db-ops/pages/write/insert-page.d.ts

@@ -15,6 +15,26 @@ import type { Knex } from 'knex';
  * anything reachable via the crawled chain must be labelled `'crawled'`
  * even if previously labelled `'inventory-*'`.
  *
+ * `is_external`, by contrast, IS overwritten on every call — but demotion is
+ * guarded. `updatePage` keys the row by the redirect DESTINATION url while
+ * passing the REQUESTING url's `isExternal`, so the value written here
+ * describes the requester, not necessarily this row. Inheriting it is
+ * deliberate when PROMOTING (an out-of-scope soft-404 page reached from an
+ * in-scope request counts as covered by the crawl, and the viewer relies on
+ * that — see `@nitpicker/query`'s `build-directory-tree-rows.ts`). It would be
+ * wrong when DEMOTING: an out-of-scope url redirecting to an in-scope page
+ * that was already taken on as a target must not flip that page to
+ * `is_external = 1` — no reading of the column justifies erasing a real
+ * observation with an inherited one. `crawler.ts`'s `#scrapedDestinations`
+ * blocks this within one run, but that is per-`#runDeal` memory, so a later
+ * `--append` / `--retry-failed` process starts blind to what the DB already
+ * knows — hence the CASE below, which checks the ROW's own prior state
+ * instead: once `scraped = 1 AND is_external = 0` is true, no later call can
+ * flip it back to `1`. Promotion (`0 → 1` before the row has been scraped, or
+ * `1 → 0` at any time) is untouched. This mirrors how `first_crawled_at`'s `COALESCE` below
+ * protects an established value — deliberately NOT by re-deriving scope from
+ * the destination url, which would also kill the wanted promoting case.
+ *
  * The page's response headers are decomposed and written into the
  * header dictionary tables here — per response, not deferred to
  * crawl-end — and the resulting `header_set_id` lands on the same

package/lib/archive/db-ops/pages/write/insert-page.js

@@ -24,6 +24,26 @@ import { upsertUrlRef } from '../../_shared/upsert-url-ref.js';
  * anything reachable via the crawled chain must be labelled `'crawled'`
  * even if previously labelled `'inventory-*'`.
  *
+ * `is_external`, by contrast, IS overwritten on every call — but demotion is
+ * guarded. `updatePage` keys the row by the redirect DESTINATION url while
+ * passing the REQUESTING url's `isExternal`, so the value written here
+ * describes the requester, not necessarily this row. Inheriting it is
+ * deliberate when PROMOTING (an out-of-scope soft-404 page reached from an
+ * in-scope request counts as covered by the crawl, and the viewer relies on
+ * that — see `@nitpicker/query`'s `build-directory-tree-rows.ts`). It would be
+ * wrong when DEMOTING: an out-of-scope url redirecting to an in-scope page
+ * that was already taken on as a target must not flip that page to
+ * `is_external = 1` — no reading of the column justifies erasing a real
+ * observation with an inherited one. `crawler.ts`'s `#scrapedDestinations`
+ * blocks this within one run, but that is per-`#runDeal` memory, so a later
+ * `--append` / `--retry-failed` process starts blind to what the DB already
+ * knows — hence the CASE below, which checks the ROW's own prior state
+ * instead: once `scraped = 1 AND is_external = 0` is true, no later call can
+ * flip it back to `1`. Promotion (`0 → 1` before the row has been scraped, or
+ * `1 → 0` at any time) is untouched. This mirrors how `first_crawled_at`'s `COALESCE` below
+ * protects an established value — deliberately NOT by re-deriving scope from
+ * the destination url, which would also kill the wanted promoting case.
+ *
  * The page's response headers are decomposed and written into the
  * header dictionary tables here — per response, not deferred to
  * crawl-end — and the resulting `header_set_id` lands on the same
@@ -86,8 +106,23 @@ export async function insertPage(knex, caches, page, isTarget, trx, source) {
         .where('id', pageId)
         .update({
         scraped: 1,
-        is_target: isTarget ? 1 : 0,
-        is_external: page.isExternal ? 1 : 0,
+        // Once a row has been scraped as a real crawl target, no later call
+        // may flip it back off — same inheritance-from-the-requester bug as
+        // `is_external` below (`setExternalPage` always passes `isTarget:
+        // false`), and the same fix: guard on the row's own prior state
+        // instead of per-run memory. Demoting is_target away from an
+        // established value would under-count `getScrapedHtmlPageCount`'s
+        // resume offset and silently break the "isTarget=1 means covered by
+        // the crawl" contract `accessor.getPages('page')` documents.
+        is_target: qb.raw('CASE WHEN scraped = 1 AND is_target = 1 THEN 1 ELSE ? END', [
+            isTarget ? 1 : 0,
+        ]),
+        // Once a row has been scraped as internal, no later call may flip it
+        // back to external — see this function's docs for why the write this
+        // guards against happens at all.
+        is_external: qb.raw('CASE WHEN scraped = 1 AND is_external = 0 THEN 0 ELSE ? END', [
+            page.isExternal ? 1 : 0,
+        ]),
         status: page.status,
         status_text: page.statusText,
         content_type_id: contentTypeId,

package/lib/archive/db-ops/pages/write/link-redirect-sources.d.ts

@@ -20,7 +20,13 @@ import type { Knex } from 'knex';
  * @param destId - `content_items.id` of the redirect destination page.
  * @param destUrlNormalized - Normalised destination URL, used to detect and
  *   skip self-redirects.
- * @param isExternal - Whether the sources are external to the crawl scope.
+ * @param isExternal - Whether the sources are external to the crawl scope, as
+ *   a single value applied to every hop in `sources` — the same
+ *   requester-describes-a-different-row shape `insertPage` guards against
+ *   (see its docs in `insert-page.ts`), since a hop can independently be a
+ *   real, already-scraped internal page in its own right (reached earlier as
+ *   a direct crawl target) before it is later observed as an intermediate hop
+ *   in some other chain. The UPDATE below carries the same CASE guard.
  * @param chainLineageSource - Lineage label propagated to each intermediate
  *   hop's row (passed through to {@link resolveContentItemId}). Derived by
  *   the caller from the **originating** page's source (`page.url`), not from

package/lib/archive/db-ops/pages/write/link-redirect-sources.js

@@ -19,7 +19,13 @@ import { resolveContentItemId } from '../../_shared/resolve-content-item-id.js';
  * @param destId - `content_items.id` of the redirect destination page.
  * @param destUrlNormalized - Normalised destination URL, used to detect and
  *   skip self-redirects.
- * @param isExternal - Whether the sources are external to the crawl scope.
+ * @param isExternal - Whether the sources are external to the crawl scope, as
+ *   a single value applied to every hop in `sources` — the same
+ *   requester-describes-a-different-row shape `insertPage` guards against
+ *   (see its docs in `insert-page.ts`), since a hop can independently be a
+ *   real, already-scraped internal page in its own right (reached earlier as
+ *   a direct crawl target) before it is later observed as an intermediate hop
+ *   in some other chain. The UPDATE below carries the same CASE guard.
  * @param chainLineageSource - Lineage label propagated to each intermediate
  *   hop's row (passed through to {@link resolveContentItemId}). Derived by
  *   the caller from the **originating** page's source (`page.url`), not from
@@ -53,7 +59,7 @@ export async function linkRedirectSources(trx, caches, sources, destId, destUrlN
             .update({
             scraped: 1,
             redirect_dest_id: destId,
-            is_external: isExternal ? 1 : 0,
+            is_external: trx.raw('CASE WHEN scraped = 1 AND is_external = 0 THEN 0 ELSE ? END', [isExternal ? 1 : 0]),
         });
         // Conditional `301 Moved Permanently` stamp — applied ONLY when the
         // row carries no definitive status yet (NULL or the `-1`

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.17.0",
+	"version": "0.18.1",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -27,10 +27,10 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/beholder": "4.1.0",
-		"@d-zero/dealer": "1.9.4",
+		"@d-zero/beholder": "4.2.2",
+		"@d-zero/dealer": "1.10.4",
 		"@d-zero/fs": "0.2.6",
-		"@d-zero/shared": "0.22.2",
+		"@d-zero/shared": "0.22.5",
 		"ansi-colors": "4.1.3",
 		"debug": "4.4.3",
 		"follow-redirects": "1.16.0",
@@ -39,7 +39,7 @@
 		"libsql": "0.5.29",
 		"puppeteer": "25.3.0",
 		"robots-parser": "3.0.1",
-		"tar": "7.5.21"
+		"tar": "7.5.22"
 	},
 	"devDependencies": {
 		"@types/debug": "4.1.13",
@@ -48,5 +48,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "2cdef7cbb4da489270e0f1c9ec2e8166b3d84e91"
+	"gitHead": "cf366bfa2ca1e0528623b3fbe1c17ac4237ca4e4"
 }