@hadoku/jobplatform-worker 2.32.37 → 2.32.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clients/scraper.d.ts +15 -0
- package/dist/clients/scraper.d.ts.map +1 -1
- package/dist/index.js +267 -68
- package/dist/ingestWrite.d.ts +41 -0
- package/dist/ingestWrite.d.ts.map +1 -0
- package/dist/routes/ingest.d.ts.map +1 -1
- package/dist/routes/jobs/importUrl.d.ts +3 -0
- package/dist/routes/jobs/importUrl.d.ts.map +1 -0
- package/dist/routes/jobs/index.d.ts.map +1 -1
- package/package.json +2 -2
|
@@ -57,5 +57,20 @@ export declare function matchCompanies(env: AppEnv, names: string[], providers?:
|
|
|
57
57
|
* worth revisiting if registry grows.
|
|
58
58
|
*/
|
|
59
59
|
export declare function triggerSearch(env: AppEnv): Promise<void>;
|
|
60
|
+
/** One posting read from its URL by the scraper, in the ingest-webhook shape. */
|
|
61
|
+
export interface ExtractedPosting {
|
|
62
|
+
job: unknown;
|
|
63
|
+
ats: string | null;
|
|
64
|
+
board_slug: string | null;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Read one job posting from a URL a person pasted (scraper
|
|
68
|
+
* `POST /api/v1/jobboards/extract`). Writes nothing on either side.
|
|
69
|
+
*
|
|
70
|
+
* Throws ScraperClientError carrying the scraper's status and its own words —
|
|
71
|
+
* "LinkedIn postings can't be imported…", "That posting no longer exists" —
|
|
72
|
+
* which are written for the person and passed through to them.
|
|
73
|
+
*/
|
|
74
|
+
export declare function extractPosting(env: AppEnv, url: string, greenhouseSlug?: string): Promise<ExtractedPosting>;
|
|
60
75
|
export {};
|
|
61
76
|
//# sourceMappingURL=scraper.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAI1C,+EAA+E;AAC/E,UAAU,WAAW;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,WAAW,EAAE,CAAC;CACpB;AAQD,8EAA8E;AAC9E,MAAM,WAAW,YAAY;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAWD,qBAAa,kBAAmB,SAAQ,KAAK;IAC5C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;gBAEV,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM;CAMzD;AA0BD;;;;;GAKG;AACH,wBAAsB,UAAU,CAC/B,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EAAE,EACf,SAAS,CAAC,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,eAAe,EAAE,CAAC,CAqB5B;AAED;;;;GAIG;AACH,wBAAsB,cAAc,CACnC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EAAE,EACf,SAAS,CAAC,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,YAAY,EAAE,CAAC,CAqBzB;AAED;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAY9D"}
|
|
1
|
+
{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAI1C,+EAA+E;AAC/E,UAAU,WAAW;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,WAAW,EAAE,CAAC;CACpB;AAQD,8EAA8E;AAC9E,MAAM,WAAW,YAAY;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAWD,qBAAa,kBAAmB,SAAQ,KAAK;IAC5C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;gBAEV,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM;CAMzD;AA0BD;;;;;GAKG;AACH,wBAAsB,UAAU,CAC/B,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EAAE,EACf,SAAS,CAAC,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,eAAe,EAAE,CAAC,CAqB5B;AAED;;;;GAIG;AACH,wBAAsB,cAAc,CACnC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EAAE,EACf,SAAS,CAAC,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,YAAY,EAAE,CAAC,CAqBzB;AAED;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAY9D;AAED,iFAAiF;AACjF,MAAM,WAAW,gBAAgB;IAChC,GAAG,EAAE,OAAO,CAAC;IACb,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAcD;;;;;;;GAOG;AACH,wBAAsB,cAAc,CACnC,GAAG,EAAE,MAAM,EACX,GAAG,EAAE,MAAM,EACX,cAAc,CAAC,EAAE,MAAM,GACrB,OAAO,CAAC,gBAAgB,CAAC,CAkB3B"}
|
package/dist/index.js
CHANGED
|
@@ -2581,6 +2581,74 @@ function scheduleRankBuild(c, db, profileId, profile) {
|
|
|
2581
2581
|
rebuildRank(db, profileId, profile).finally(() => building.delete(profileId))
|
|
2582
2582
|
);
|
|
2583
2583
|
}
|
|
2584
|
+
function normalizeWorkplaceType(wt) {
|
|
2585
|
+
if (wt === "on_site") return "onsite";
|
|
2586
|
+
return wt;
|
|
2587
|
+
}
|
|
2588
|
+
function insertJobStatement(db, job, { source, board_slug, now }) {
|
|
2589
|
+
const workplaceType = normalizeWorkplaceType(job.workplace_type);
|
|
2590
|
+
const structuredMin = job.salary?.min ?? null;
|
|
2591
|
+
const structuredMax = job.salary?.max ?? null;
|
|
2592
|
+
const prose = structuredMin === null && structuredMax === null ? parseSalaryRange(job.description) : null;
|
|
2593
|
+
const salaryMin = structuredMin ?? prose?.min ?? null;
|
|
2594
|
+
const salaryMax = structuredMax ?? prose?.max ?? null;
|
|
2595
|
+
const parsed = parseAtsSlug(job.url);
|
|
2596
|
+
const ats = parsed.ats ?? (board_slug ? source : null);
|
|
2597
|
+
const slug2 = board_slug ?? parsed.slug;
|
|
2598
|
+
const slugSource = slug2 === null ? null : board_slug ? "scraped" : "guessed";
|
|
2599
|
+
const role = classifyRole(job.title, job.description);
|
|
2600
|
+
return db.prepare(
|
|
2601
|
+
`INSERT OR IGNORE INTO jobs (
|
|
2602
|
+
id, url, source_site, title, company, location,
|
|
2603
|
+
job_type, workplace_type, salary_min, salary_max,
|
|
2604
|
+
description, posted_date, application_url, department,
|
|
2605
|
+
scraper_used, raw, scraped_at, last_seen_at, ats, slug,
|
|
2606
|
+
role_track, role_level, slug_source, geo_keys
|
|
2607
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
2608
|
+
).bind(
|
|
2609
|
+
job.id,
|
|
2610
|
+
job.url,
|
|
2611
|
+
job.source_site,
|
|
2612
|
+
job.title,
|
|
2613
|
+
job.company,
|
|
2614
|
+
job.location,
|
|
2615
|
+
job.job_type,
|
|
2616
|
+
workplaceType,
|
|
2617
|
+
salaryMin,
|
|
2618
|
+
salaryMax,
|
|
2619
|
+
job.description,
|
|
2620
|
+
job.posted_date ?? null,
|
|
2621
|
+
job.application_url ?? null,
|
|
2622
|
+
job.department ?? null,
|
|
2623
|
+
job.scraper_used ?? null,
|
|
2624
|
+
JSON.stringify(job.raw),
|
|
2625
|
+
now,
|
|
2626
|
+
now,
|
|
2627
|
+
ats,
|
|
2628
|
+
slug2,
|
|
2629
|
+
role.track,
|
|
2630
|
+
role.level,
|
|
2631
|
+
slugSource,
|
|
2632
|
+
geoKeys(job.location)
|
|
2633
|
+
);
|
|
2634
|
+
}
|
|
2635
|
+
async function rankNewJobs(db) {
|
|
2636
|
+
const ranked = await db.prepare("SELECT profile_id, criteria_hash, max_job_rowid FROM job_profile_rank_state").all();
|
|
2637
|
+
if (ranked.results.length === 0) return;
|
|
2638
|
+
const head2 = await jobsHead(db);
|
|
2639
|
+
for (const state of ranked.results) {
|
|
2640
|
+
const from = Number(state.max_job_rowid);
|
|
2641
|
+
if (from >= head2) continue;
|
|
2642
|
+
const profile = await loadScorableProfile(db, state.profile_id);
|
|
2643
|
+
if (criteriaHash(profile) !== state.criteria_hash) continue;
|
|
2644
|
+
const rows = await db.prepare(
|
|
2645
|
+
`SELECT id, title, location, workplace_type, salary_max, role_level
|
|
2646
|
+
FROM jobs WHERE rowid > ? AND rowid <= ?`
|
|
2647
|
+
).bind(from, head2).all();
|
|
2648
|
+
await writeRanks(db, state.profile_id, profile, rows.results);
|
|
2649
|
+
await markRankBuilt(db, state.profile_id, profile, head2);
|
|
2650
|
+
}
|
|
2651
|
+
}
|
|
2584
2652
|
const app$4 = new OpenAPIHono();
|
|
2585
2653
|
const CULL_DAYS = 60;
|
|
2586
2654
|
const CULL_BATCH = 500;
|
|
@@ -2614,23 +2682,6 @@ async function stripStaleDescriptions(db) {
|
|
|
2614
2682
|
).bind(cutoff, cutoff).run();
|
|
2615
2683
|
return res.meta?.changes ?? 0;
|
|
2616
2684
|
}
|
|
2617
|
-
async function rankNewJobs(db) {
|
|
2618
|
-
const ranked = await db.prepare("SELECT profile_id, criteria_hash, max_job_rowid FROM job_profile_rank_state").all();
|
|
2619
|
-
if (ranked.results.length === 0) return;
|
|
2620
|
-
const head2 = await jobsHead(db);
|
|
2621
|
-
for (const state of ranked.results) {
|
|
2622
|
-
const from = Number(state.max_job_rowid);
|
|
2623
|
-
if (from >= head2) continue;
|
|
2624
|
-
const profile = await loadScorableProfile(db, state.profile_id);
|
|
2625
|
-
if (criteriaHash(profile) !== state.criteria_hash) continue;
|
|
2626
|
-
const rows = await db.prepare(
|
|
2627
|
-
`SELECT id, title, location, workplace_type, salary_max, role_level
|
|
2628
|
-
FROM jobs WHERE rowid > ? AND rowid <= ?`
|
|
2629
|
-
).bind(from, head2).all();
|
|
2630
|
-
await writeRanks(db, state.profile_id, profile, rows.results);
|
|
2631
|
-
await markRankBuilt(db, state.profile_id, profile, head2);
|
|
2632
|
-
}
|
|
2633
|
-
}
|
|
2634
2685
|
app$4.use("/ingest", requireMinTier("friend"));
|
|
2635
2686
|
app$4.use("/ingest/backfill-slugs", requireMinTier("friend"));
|
|
2636
2687
|
app$4.use("/ingest/backfill-roles", requireMinTier("friend"));
|
|
@@ -2640,10 +2691,6 @@ app$4.use("/ingest/cull", requireMinTier("friend"));
|
|
|
2640
2691
|
app$4.use("/ingest/rebuild-rank", requireMinTier("friend"));
|
|
2641
2692
|
app$4.use("/ingest/reconcile-mail", requireMinTier("friend"));
|
|
2642
2693
|
app$4.use("/directives", requireMinTier("friend"));
|
|
2643
|
-
function normalizeWorkplaceType(wt) {
|
|
2644
|
-
if (wt === "on_site") return "onsite";
|
|
2645
|
-
return wt;
|
|
2646
|
-
}
|
|
2647
2694
|
const ingestRoute = createRoute({
|
|
2648
2695
|
method: "post",
|
|
2649
2696
|
path: "/ingest",
|
|
@@ -2667,53 +2714,7 @@ app$4.openapi(ingestRoute, async (c) => {
|
|
|
2667
2714
|
const { jobs, batch_number, is_final, source, board_slug } = c.req.valid("json");
|
|
2668
2715
|
const db = c.env.JOB_PLATFORM_DB;
|
|
2669
2716
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2670
|
-
const stmts = jobs.map((job) => {
|
|
2671
|
-
const workplaceType = normalizeWorkplaceType(job.workplace_type);
|
|
2672
|
-
const structuredMin = job.salary?.min ?? null;
|
|
2673
|
-
const structuredMax = job.salary?.max ?? null;
|
|
2674
|
-
const prose = structuredMin === null && structuredMax === null ? parseSalaryRange(job.description) : null;
|
|
2675
|
-
const salaryMin = structuredMin ?? prose?.min ?? null;
|
|
2676
|
-
const salaryMax = structuredMax ?? prose?.max ?? null;
|
|
2677
|
-
const parsed = parseAtsSlug(job.url);
|
|
2678
|
-
const ats = parsed.ats ?? (board_slug ? source : null);
|
|
2679
|
-
const slug2 = board_slug ?? parsed.slug;
|
|
2680
|
-
const slugSource = slug2 === null ? null : board_slug ? "scraped" : "guessed";
|
|
2681
|
-
const role = classifyRole(job.title, job.description);
|
|
2682
|
-
return db.prepare(
|
|
2683
|
-
`INSERT OR IGNORE INTO jobs (
|
|
2684
|
-
id, url, source_site, title, company, location,
|
|
2685
|
-
job_type, workplace_type, salary_min, salary_max,
|
|
2686
|
-
description, posted_date, application_url, department,
|
|
2687
|
-
scraper_used, raw, scraped_at, last_seen_at, ats, slug,
|
|
2688
|
-
role_track, role_level, slug_source, geo_keys
|
|
2689
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
2690
|
-
).bind(
|
|
2691
|
-
job.id,
|
|
2692
|
-
job.url,
|
|
2693
|
-
job.source_site,
|
|
2694
|
-
job.title,
|
|
2695
|
-
job.company,
|
|
2696
|
-
job.location,
|
|
2697
|
-
job.job_type,
|
|
2698
|
-
workplaceType,
|
|
2699
|
-
salaryMin,
|
|
2700
|
-
salaryMax,
|
|
2701
|
-
job.description,
|
|
2702
|
-
job.posted_date ?? null,
|
|
2703
|
-
job.application_url ?? null,
|
|
2704
|
-
job.department ?? null,
|
|
2705
|
-
job.scraper_used ?? null,
|
|
2706
|
-
JSON.stringify(job.raw),
|
|
2707
|
-
now,
|
|
2708
|
-
now,
|
|
2709
|
-
ats,
|
|
2710
|
-
slug2,
|
|
2711
|
-
role.track,
|
|
2712
|
-
role.level,
|
|
2713
|
-
slugSource,
|
|
2714
|
-
geoKeys(job.location)
|
|
2715
|
-
);
|
|
2716
|
-
});
|
|
2717
|
+
const stmts = jobs.map((job) => insertJobStatement(db, job, { source, board_slug, now }));
|
|
2717
2718
|
const inserted = [];
|
|
2718
2719
|
startTime(c, "insert");
|
|
2719
2720
|
for (let i = 0; i < stmts.length; i += INGEST_BATCH_CHUNK) {
|
|
@@ -3401,6 +3402,34 @@ async function triggerSearch(env) {
|
|
|
3401
3402
|
);
|
|
3402
3403
|
}
|
|
3403
3404
|
}
|
|
3405
|
+
function scraperMessage(body) {
|
|
3406
|
+
try {
|
|
3407
|
+
const parsed = JSON.parse(body);
|
|
3408
|
+
if (typeof parsed.detail === "string") return parsed.detail;
|
|
3409
|
+
if (typeof parsed.error?.message === "string") return parsed.error.message;
|
|
3410
|
+
} catch {
|
|
3411
|
+
}
|
|
3412
|
+
return null;
|
|
3413
|
+
}
|
|
3414
|
+
async function extractPosting(env, url, greenhouseSlug) {
|
|
3415
|
+
const response = await scraperFetch(env, "/api/v1/jobboards/extract", {
|
|
3416
|
+
method: "POST",
|
|
3417
|
+
body: JSON.stringify(greenhouseSlug ? { url, greenhouse_slug: greenhouseSlug } : { url })
|
|
3418
|
+
});
|
|
3419
|
+
const text = await response.text();
|
|
3420
|
+
if (!response.ok) {
|
|
3421
|
+
throw new ScraperClientError(
|
|
3422
|
+
scraperMessage(text) ?? `scraper /extract returned ${response.status}`,
|
|
3423
|
+
response.status,
|
|
3424
|
+
text
|
|
3425
|
+
);
|
|
3426
|
+
}
|
|
3427
|
+
const json = JSON.parse(text);
|
|
3428
|
+
if (!json.success || !json.data) {
|
|
3429
|
+
throw new ScraperClientError("scraper /extract reported failure", 502, text);
|
|
3430
|
+
}
|
|
3431
|
+
return json.data;
|
|
3432
|
+
}
|
|
3404
3433
|
const app$3 = new OpenAPIHono();
|
|
3405
3434
|
app$3.use("/profiles", requireMinTier("friend"));
|
|
3406
3435
|
app$3.use("/profiles/*", requireMinTier("friend"));
|
|
@@ -7106,6 +7135,175 @@ function registerLocationsRoute(app2) {
|
|
|
7106
7135
|
}
|
|
7107
7136
|
);
|
|
7108
7137
|
}
|
|
7138
|
+
const ImportJobSchema = z.object({
|
|
7139
|
+
url: z.string().trim().url().max(2e3),
|
|
7140
|
+
/** Where it lands in the pipeline when it is new to the caller. */
|
|
7141
|
+
state: JobStateSchema.exclude(["dismissed"]).default("saved")
|
|
7142
|
+
}).openapi("ImportJob");
|
|
7143
|
+
const ImportedSchema = z.object({
|
|
7144
|
+
success: z.literal(true),
|
|
7145
|
+
data: z.object({
|
|
7146
|
+
job: z.object({
|
|
7147
|
+
id: z.string(),
|
|
7148
|
+
title: z.string(),
|
|
7149
|
+
company: z.string(),
|
|
7150
|
+
location: z.string(),
|
|
7151
|
+
url: z.string()
|
|
7152
|
+
}),
|
|
7153
|
+
/** The posting was new to the corpus (false: it was already there). */
|
|
7154
|
+
created: z.boolean(),
|
|
7155
|
+
/** The caller's state for it now. */
|
|
7156
|
+
state: JobStateSchema,
|
|
7157
|
+
/** The caller already tracked it, so its state was left alone. */
|
|
7158
|
+
already_tracked: z.boolean()
|
|
7159
|
+
})
|
|
7160
|
+
}).openapi("ImportedJob");
|
|
7161
|
+
function failure(status, message) {
|
|
7162
|
+
return {
|
|
7163
|
+
success: false,
|
|
7164
|
+
error: status === 422 ? "Unprocessable" : "Bad gateway",
|
|
7165
|
+
message
|
|
7166
|
+
};
|
|
7167
|
+
}
|
|
7168
|
+
const JOB_COLS = "id, title, company, location, url";
|
|
7169
|
+
async function greenhouseSlugFor(db, url) {
|
|
7170
|
+
const parsed = new URL(url);
|
|
7171
|
+
if (!parsed.searchParams.has("gh_jid")) return void 0;
|
|
7172
|
+
const prefix = `${parsed.protocol}//${parsed.host}/`;
|
|
7173
|
+
const row = await db.prepare(
|
|
7174
|
+
`SELECT slug FROM jobs
|
|
7175
|
+
WHERE url >= ? AND url < ? AND ats = 'greenhouse' AND slug IS NOT NULL
|
|
7176
|
+
AND slug_source = 'scraped'
|
|
7177
|
+
LIMIT 1`
|
|
7178
|
+
).bind(prefix, `${prefix.slice(0, -1)}0`).first();
|
|
7179
|
+
return row?.slug;
|
|
7180
|
+
}
|
|
7181
|
+
function registerImportRoute(app2) {
|
|
7182
|
+
app2.post("/jobs/import", gateAuthed);
|
|
7183
|
+
app2.openapi(
|
|
7184
|
+
createRoute({
|
|
7185
|
+
method: "post",
|
|
7186
|
+
path: "/jobs/import",
|
|
7187
|
+
tags: ["Pipeline"],
|
|
7188
|
+
summary: "Import a posting from its URL into the corpus and the caller’s pipeline",
|
|
7189
|
+
request: {
|
|
7190
|
+
query: ownerNameQuery,
|
|
7191
|
+
body: { content: { "application/json": { schema: ImportJobSchema } } }
|
|
7192
|
+
},
|
|
7193
|
+
responses: {
|
|
7194
|
+
200: {
|
|
7195
|
+
description: "Imported (or found, and added to the pipeline)",
|
|
7196
|
+
content: { "application/json": { schema: ImportedSchema } }
|
|
7197
|
+
},
|
|
7198
|
+
403: {
|
|
7199
|
+
description: "Forbidden",
|
|
7200
|
+
content: { "application/json": { schema: IdentityErrorResponseSchema } }
|
|
7201
|
+
},
|
|
7202
|
+
404: {
|
|
7203
|
+
description: "The board no longer lists that posting, or no such owner name",
|
|
7204
|
+
content: { "application/json": { schema: IdentityErrorResponseSchema } }
|
|
7205
|
+
},
|
|
7206
|
+
409: {
|
|
7207
|
+
description: "That owner name has never signed in",
|
|
7208
|
+
content: { "application/json": { schema: IdentityErrorResponseSchema } }
|
|
7209
|
+
},
|
|
7210
|
+
422: {
|
|
7211
|
+
description: "No posting could be read from that page (or it is a LinkedIn URL)",
|
|
7212
|
+
content: { "application/json": { schema: ErrorResponseSchema } }
|
|
7213
|
+
},
|
|
7214
|
+
502: {
|
|
7215
|
+
description: "The site, or the scraper, could not be reached",
|
|
7216
|
+
content: { "application/json": { schema: ErrorResponseSchema } }
|
|
7217
|
+
},
|
|
7218
|
+
503: {
|
|
7219
|
+
description: "Identity could not be resolved right now — retry",
|
|
7220
|
+
content: { "application/json": { schema: IdentityErrorResponseSchema } }
|
|
7221
|
+
}
|
|
7222
|
+
}
|
|
7223
|
+
}),
|
|
7224
|
+
async (c) => {
|
|
7225
|
+
const { url, state } = c.req.valid("json");
|
|
7226
|
+
const { ownerName } = c.req.valid("query");
|
|
7227
|
+
const who = await effectiveUserId(c, ownerName);
|
|
7228
|
+
if (isEffectiveUserError(who)) return c.json(who.error.body, who.error.status);
|
|
7229
|
+
const db = c.env.JOB_PLATFORM_DB;
|
|
7230
|
+
let row = await db.prepare(`SELECT ${JOB_COLS} FROM jobs WHERE url = ?`).bind(url).first();
|
|
7231
|
+
let created = false;
|
|
7232
|
+
if (!row) {
|
|
7233
|
+
let extracted;
|
|
7234
|
+
try {
|
|
7235
|
+
extracted = await extractPosting(c.env, url, await greenhouseSlugFor(db, url));
|
|
7236
|
+
} catch (err) {
|
|
7237
|
+
if (!(err instanceof ScraperClientError)) throw err;
|
|
7238
|
+
if (err.status === 404) {
|
|
7239
|
+
return c.json(
|
|
7240
|
+
{ success: false, error: "Not found", message: err.message },
|
|
7241
|
+
404
|
|
7242
|
+
);
|
|
7243
|
+
}
|
|
7244
|
+
if (err.status === 400 || err.status === 422) {
|
|
7245
|
+
return c.json(failure(422, err.message), 422);
|
|
7246
|
+
}
|
|
7247
|
+
return c.json(failure(502, `Could not read that posting: ${err.message}`), 502);
|
|
7248
|
+
}
|
|
7249
|
+
const job = IngestJobSchema.safeParse(extracted.job);
|
|
7250
|
+
if (!job.success) {
|
|
7251
|
+
return c.json(
|
|
7252
|
+
failure(
|
|
7253
|
+
502,
|
|
7254
|
+
`The scraper returned a posting this API cannot store: ${job.error.message}`
|
|
7255
|
+
),
|
|
7256
|
+
502
|
|
7257
|
+
);
|
|
7258
|
+
}
|
|
7259
|
+
const now2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
7260
|
+
const [res] = await db.batch([
|
|
7261
|
+
insertJobStatement(db, job.data, {
|
|
7262
|
+
source: extracted.ats ?? "import",
|
|
7263
|
+
board_slug: extracted.board_slug ?? void 0,
|
|
7264
|
+
now: now2
|
|
7265
|
+
})
|
|
7266
|
+
]);
|
|
7267
|
+
created = (res.meta?.changes ?? 0) > 0;
|
|
7268
|
+
row = await db.prepare(`SELECT ${JOB_COLS} FROM jobs WHERE url = ? OR id = ? LIMIT 1`).bind(job.data.url, job.data.id).first();
|
|
7269
|
+
if (!row) throw new Error(`import: ${job.data.id} neither inserted nor found`);
|
|
7270
|
+
if (created) await rankNewJobs(db);
|
|
7271
|
+
}
|
|
7272
|
+
const prev = await db.prepare("SELECT state FROM job_states WHERE job_id = ? AND user_id = ?").bind(row.id, who.userId).first();
|
|
7273
|
+
if (prev) {
|
|
7274
|
+
const kept = JobStateSchema.safeParse(prev.state);
|
|
7275
|
+
return c.json(
|
|
7276
|
+
{
|
|
7277
|
+
success: true,
|
|
7278
|
+
data: {
|
|
7279
|
+
job: row,
|
|
7280
|
+
created,
|
|
7281
|
+
state: kept.success ? kept.data : state,
|
|
7282
|
+
already_tracked: true
|
|
7283
|
+
}
|
|
7284
|
+
},
|
|
7285
|
+
200
|
|
7286
|
+
);
|
|
7287
|
+
}
|
|
7288
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
7289
|
+
await db.batch([
|
|
7290
|
+
db.prepare(
|
|
7291
|
+
`INSERT INTO job_events (job_id, user_id, kind, from_state, to_state, occurred_at, created_at)
|
|
7292
|
+
SELECT ?, ?, 'state', NULL, ?, ?, ?
|
|
7293
|
+
WHERE NOT EXISTS (SELECT 1 FROM job_states WHERE job_id = ? AND user_id = ?)`
|
|
7294
|
+
).bind(row.id, who.userId, state, now, now, row.id, who.userId),
|
|
7295
|
+
db.prepare(
|
|
7296
|
+
`INSERT INTO job_states (job_id, user_id, state, updated_at) VALUES (?, ?, ?, ?)
|
|
7297
|
+
ON CONFLICT (job_id, user_id) DO NOTHING`
|
|
7298
|
+
).bind(row.id, who.userId, state, now)
|
|
7299
|
+
]);
|
|
7300
|
+
return c.json(
|
|
7301
|
+
{ success: true, data: { job: row, created, state, already_tracked: false } },
|
|
7302
|
+
200
|
|
7303
|
+
);
|
|
7304
|
+
}
|
|
7305
|
+
);
|
|
7306
|
+
}
|
|
7109
7307
|
const app$2 = new OpenAPIHono();
|
|
7110
7308
|
registerFeedRoute(app$2);
|
|
7111
7309
|
registerPreflightRoute(app$2);
|
|
@@ -7121,6 +7319,7 @@ registerFillRoutes(app$2);
|
|
|
7121
7319
|
registerBlockedRoutes(app$2);
|
|
7122
7320
|
registerInsightsRoute(app$2);
|
|
7123
7321
|
registerLocationsRoute(app$2);
|
|
7322
|
+
registerImportRoute(app$2);
|
|
7124
7323
|
registerAnswerRoutes(app$2);
|
|
7125
7324
|
registerOwnerRoutes(app$2);
|
|
7126
7325
|
const jobRoutes = app$2;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Writing postings into the corpus — the one path both POST /ingest (the
|
|
3
|
+
* scraper's nightly batches) and POST /jobs/import (one posting a person
|
|
4
|
+
* pasted) go through, so an imported posting is classified, salaried, placed
|
|
5
|
+
* and ranked exactly as a scraped one is.
|
|
6
|
+
*/
|
|
7
|
+
import type { z } from '@hono/zod-openapi';
|
|
8
|
+
import type { AppEnv } from './types.js';
|
|
9
|
+
import type { IngestJobSchema } from './schemas.js';
|
|
10
|
+
type DB = AppEnv['JOB_PLATFORM_DB'];
|
|
11
|
+
export type IngestJob = z.infer<typeof IngestJobSchema>;
|
|
12
|
+
/**
|
|
13
|
+
* The INSERT OR IGNORE for one posting. Ignored when its url (UNIQUE) or id
|
|
14
|
+
* is already stored — an already-known posting keeps the fields it has.
|
|
15
|
+
*
|
|
16
|
+
* `source` is the batch label; `board_slug` the board the scraper actually
|
|
17
|
+
* fetched, when it says so.
|
|
18
|
+
*/
|
|
19
|
+
export declare function insertJobStatement(db: DB, job: IngestJob, { source, board_slug, now }: {
|
|
20
|
+
source: string;
|
|
21
|
+
board_slug?: string;
|
|
22
|
+
now: string;
|
|
23
|
+
}): import("@cloudflare/workers-types").D1PreparedStatement;
|
|
24
|
+
/**
|
|
25
|
+
* Extend every current ranking to cover the jobs inserted since it was built.
|
|
26
|
+
*
|
|
27
|
+
* Selected by rowid past each profile's marker rather than from the payload:
|
|
28
|
+
* ingest only ever INSERTs (an already-known posting is ignored, its fields
|
|
29
|
+
* untouched), so a job that was ranked once keeps the same bound and needs no
|
|
30
|
+
* new write. Re-ranking the whole payload whenever any of it was new used to
|
|
31
|
+
* upsert ~25 rows per profile per batch for one new job — 35% of the
|
|
32
|
+
* database's daily writes. The rowid range also picks up jobs a previous,
|
|
33
|
+
* failed extension missed, which a payload-only pass never would.
|
|
34
|
+
*
|
|
35
|
+
* A profile whose criteria changed since its ranking was built is left alone:
|
|
36
|
+
* extending it under the new criteria and moving its marker would bless the
|
|
37
|
+
* old rows as current. The feed sees the stale hash and rebuilds it.
|
|
38
|
+
*/
|
|
39
|
+
export declare function rankNewJobs(db: AppEnv['JOB_PLATFORM_DB']): Promise<void>;
|
|
40
|
+
export {};
|
|
41
|
+
//# sourceMappingURL=ingestWrite.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ingestWrite.d.ts","sourceRoot":"","sources":["../src/ingestWrite.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAQpD,KAAK,EAAE,GAAG,MAAM,CAAC,iBAAiB,CAAC,CAAC;AACpC,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAQxD;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CACjC,EAAE,EAAE,EAAE,EACN,GAAG,EAAE,SAAS,EACd,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,2DA6DjF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,WAAW,CAAC,EAAE,EAAE,MAAM,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAyB9E"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/routes/ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAkB,MAAM,mBAAmB,CAAC;AAEhE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAkB,KAAK,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/routes/ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAkB,MAAM,mBAAmB,CAAC;AAEhE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAkB,KAAK,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAiB9E,UAAU,YAAY;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE;QAAE,WAAW,EAAE,iBAAiB,CAAA;KAAE,CAAC;CAC9C;AAs7BD,eAAO,MAAM,YAAY,oCAAM,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"importUrl.d.ts","sourceRoot":"","sources":["../../../src/routes/jobs/importUrl.ts"],"names":[],"mappings":"AA2BA,OAAO,EAKN,KAAK,OAAO,EACZ,MAAM,aAAa,CAAC;AA2ErB,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAuJtD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/routes/jobs/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/routes/jobs/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAyChD,eAAO,MAAM,SAAS,oCAAM,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hadoku/jobplatform-worker",
|
|
3
|
-
"version": "2.32.
|
|
4
|
-
"gitHead": "
|
|
3
|
+
"version": "2.32.38",
|
|
4
|
+
"gitHead": "4f79b4c6dc42dbdc2063fd7911c625e4a754b007",
|
|
5
5
|
"description": "Job listings aggregation and tracking dashboard - Worker package",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"type": "module",
|