@hadoku/jobplatform-worker 2.32.38 → 2.32.39

package/dist/index.js

@@ -2725,20 +2725,34 @@ app$4.openapi(ingestRoute, async (c) => {
   const accepted = inserted.filter(Boolean).length;
   const skipped = jobs.length - accepted;
   const seen = jobs.filter((_, i) => !inserted[i]);
-  const seenUrls = seen.map((j) => j.url);
-  const seenStmts = [];
-  const strippedStmts = [];
-  for (let i = 0; i < seenUrls.length; i += 90) {
-    const chunk = seenUrls.slice(i, i + 90);
+  const SEEN_CHUNK = 45;
+  const storedReads = [];
+  for (let i = 0; i < seen.length; i += SEEN_CHUNK) {
+    const chunk = seen.slice(i, i + SEEN_CHUNK);
     const inList = chunk.map(() => "?").join(",");
-    seenStmts.push(
-      db.prepare(`UPDATE jobs SET last_seen_at = ? WHERE url IN (${inList})`).bind(now, ...chunk)
+    storedReads.push(
+      db.prepare(
+        `SELECT id, url, description = '' AS blank FROM jobs
+					 WHERE url IN (${inList}) OR id IN (${inList})`
+      ).bind(...chunk.map((j) => j.url), ...chunk.map((j) => j.id))
     );
-    strippedStmts.push(
-      db.prepare(`SELECT url FROM jobs WHERE url IN (${inList}) AND description = ''`).bind(...chunk)
+  }
+  startTime(c, "seen");
+  const stored = storedReads.length ? (await db.batch(storedReads)).flatMap(
+    (r) => r.results
+  ) : [];
+  const byUrl = new Map(stored.map((r) => [r.url, r]));
+  const byId = new Map(stored.map((r) => [r.id, r]));
+  const writes = [];
+  const sameUrl = seen.filter((j) => byUrl.has(j.url)).map((j) => j.url);
+  for (let i = 0; i < sameUrl.length; i += 90) {
+    const chunk = sameUrl.slice(i, i + 90);
+    const inList = chunk.map(() => "?").join(",");
+    writes.push(
+      db.prepare(`UPDATE jobs SET last_seen_at = ? WHERE url IN (${inList})`).bind(now, ...chunk)
     );
     if (board_slug) {
-      seenStmts.push(
+      writes.push(
         db.prepare(
           `UPDATE jobs SET slug = ?, slug_source = 'scraped'
 						 WHERE url IN (${inList})
@@ -2747,21 +2761,24 @@ app$4.openapi(ingestRoute, async (c) => {
       );
     }
   }
-  startTime(c, "seen");
-  if (seenStmts.length) await db.batch(seenStmts);
-  endTime(c, "seen");
-  startTime(c, "restore");
-  if (strippedStmts.length) {
-    const found = await db.batch(strippedStmts);
-    const stripped = new Set(found.flatMap((r) => r.results.map((row) => row.url)));
-    const restores = seen.filter((j) => stripped.has(j.url) && j.description !== "").map(
-      (j) => db.prepare(`UPDATE jobs SET description = ? WHERE url = ? AND description = ''`).bind(j.description, j.url)
-    );
-    for (let i = 0; i < restores.length; i += INGEST_BATCH_CHUNK) {
-      await db.batch(restores.slice(i, i + INGEST_BATCH_CHUNK));
+  for (const j of seen) {
+    const row = byId.get(j.id);
+    if (!row || row.url === j.url) continue;
+    writes.push(db.prepare("UPDATE jobs SET last_seen_at = ? WHERE id = ?").bind(now, row.id));
+    writes.push(db.prepare("UPDATE OR IGNORE jobs SET url = ? WHERE id = ?").bind(j.url, row.id));
+  }
+  for (const j of seen) {
+    const row = byUrl.get(j.url) ?? byId.get(j.id);
+    if (row?.blank && j.description !== "") {
+      writes.push(
+        db.prepare(`UPDATE jobs SET description = ? WHERE id = ? AND description = ''`).bind(j.description, row.id)
+      );
     }
   }
-  endTime(c, "restore");
+  for (let i = 0; i < writes.length; i += INGEST_BATCH_CHUNK) {
+    await db.batch(writes.slice(i, i + INGEST_BATCH_CHUNK));
+  }
+  endTime(c, "seen");
   if (accepted > 0) {
     startTime(c, "rank");
     await rankNewJobs(db);

package/dist/routes/ingest.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/routes/ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAkB,MAAM,mBAAmB,CAAC;AAEhE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAkB,KAAK,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAiB9E,UAAU,YAAY;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE;QAAE,WAAW,EAAE,iBAAiB,CAAA;KAAE,CAAC;CAC9C;AAs7BD,eAAO,MAAM,YAAY,oCAAM,CAAC"}
+{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/routes/ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAkB,MAAM,mBAAmB,CAAC;AAEhE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAkB,KAAK,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAiB9E,UAAU,YAAY;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE;QAAE,WAAW,EAAE,iBAAiB,CAAA;KAAE,CAAC;CAC9C;AAq9BD,eAAO,MAAM,YAAY,oCAAM,CAAC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@hadoku/jobplatform-worker",
-  "version": "2.32.38",
-  "gitHead": "4f79b4c6dc42dbdc2063fd7911c625e4a754b007",
+  "version": "2.32.39",
+  "gitHead": "84b5623c402aac9159148daa3d454529dfea4407",
   "description": "Job listings aggregation and tracking dashboard - Worker package",
   "license": "MIT",
   "type": "module",