jobcrawl 0.1.0

package/dist/core/match-jobs.js

@@ -0,0 +1,43 @@
+/**
+ * Filter jobs against search criteria. Case-insensitive substring matching.
+ * Jobs with null fields are included (don't exclude for missing data).
+ */
+export function matchJobs(jobs, criteria) {
+    return jobs.filter((job) => {
+        // Keywords: any keyword must match title (include if no keywords specified)
+        if (criteria.keywords.length > 0) {
+            const title = job.title.toLowerCase();
+            const hasMatch = criteria.keywords.some((kw) => title.includes(kw.toLowerCase()));
+            if (!hasMatch)
+                return false;
+        }
+        // Exclude keywords: exclude if any match title
+        if (criteria.excludeKeywords.length > 0) {
+            const title = job.title.toLowerCase();
+            const hasExclude = criteria.excludeKeywords.some((kw) => title.includes(kw.toLowerCase()));
+            if (hasExclude)
+                return false;
+        }
+        // Location: substring match (include if job has no location)
+        if (criteria.location && job.location) {
+            if (!job.location.toLowerCase().includes(criteria.location.toLowerCase())) {
+                return false;
+            }
+        }
+        // Work mode: include if job's workMode is in allowed set (include if null)
+        if (criteria.workMode && criteria.workMode.length > 0 && job.workMode) {
+            if (!criteria.workMode.includes(job.workMode))
+                return false;
+        }
+        // Departments: substring match (include if null)
+        if (criteria.departments &&
+            criteria.departments.length > 0 &&
+            job.department) {
+            const dept = job.department.toLowerCase();
+            const hasMatch = criteria.departments.some((d) => dept.includes(d.toLowerCase()));
+            if (!hasMatch)
+                return false;
+        }
+        return true;
+    });
+}

package/dist/core/providers/ashby.d.ts

@@ -0,0 +1,6 @@
+import type { Job, SearchCriteria } from "../../types/index.js";
+/**
+ * Fetch jobs from Ashby's public posting API.
+ * No auth required. Returns all jobs as a single dump.
+ */
+export declare function extractAshbyJobs(boardToken: string, sourceUrl: string, _criteria: SearchCriteria, saveRaw?: boolean): Promise<Job[]>;

package/dist/core/providers/ashby.js

@@ -0,0 +1,58 @@
+import { createHash } from "node:crypto";
+import { saveRawResponse } from "../save-raw.js";
+/**
+ * Fetch jobs from Ashby's public posting API.
+ * No auth required. Returns all jobs as a single dump.
+ */
+export async function extractAshbyJobs(boardToken, sourceUrl, _criteria, saveRaw = false) {
+    const apiUrl = `https://api.ashbyhq.com/posting-api/job-board/${boardToken}`;
+    const response = await fetch(apiUrl);
+    if (!response.ok) {
+        throw new Error(`Ashby API returned ${response.status} for board "${boardToken}"`);
+    }
+    const data = (await response.json());
+    if (saveRaw)
+        await saveRawResponse("ashby", boardToken, data);
+    return data.jobs.map((aj) => {
+        const id = createHash("sha256")
+            .update(`${aj.jobUrl}:${aj.title}:${boardToken}`)
+            .digest("hex")
+            .slice(0, 12);
+        return {
+            id,
+            title: aj.title,
+            company: boardToken,
+            location: aj.location ?? null,
+            workMode: inferWorkMode(aj),
+            department: aj.department ?? aj.team ?? null,
+            url: aj.jobUrl,
+            sourceUrl,
+            provider: "ashby",
+            description: aj.descriptionPlain?.slice(0, 200) ?? null,
+            postedAt: aj.publishedAt ?? null,
+            extractedAt: new Date().toISOString(),
+            raw: aj,
+        };
+    });
+}
+function inferWorkMode(job) {
+    if (job.isRemote === true)
+        return "remote";
+    if (job.workplaceType) {
+        const wt = job.workplaceType.toLowerCase();
+        if (wt.includes("remote"))
+            return "remote";
+        if (wt.includes("hybrid"))
+            return "hybrid";
+        if (wt.includes("onsite") || wt.includes("on-site"))
+            return "onsite";
+    }
+    if (job.location) {
+        const loc = job.location.toLowerCase();
+        if (loc.includes("remote"))
+            return "remote";
+        if (loc.includes("hybrid"))
+            return "hybrid";
+    }
+    return null;
+}

package/dist/core/providers/generic.d.ts

@@ -0,0 +1,6 @@
+import type { Job } from "../../types/index.js";
+/**
+ * Heuristic extraction from rendered HTML.
+ * Two strategies: (1) job-like <a> links, (2) repeated container elements.
+ */
+export declare function extractGenericJobs(html: string, sourceUrl: string): Job[];

package/dist/core/providers/generic.js

@@ -0,0 +1,294 @@
+import { createHash } from "node:crypto";
+import { parse } from "node-html-parser";
+/**
+ * Heuristic extraction from rendered HTML.
+ * Two strategies: (1) job-like <a> links, (2) repeated container elements.
+ */
+export function extractGenericJobs(html, sourceUrl) {
+    const root = parse(html);
+    const canonical = normalizeUrl(sourceUrl);
+    // Strategy 1: Link-based extraction
+    const linkJobs = extractFromLinks(root, sourceUrl);
+    const realLinkJobs = linkJobs.filter((j) => normalizeUrl(j.url) !== canonical);
+    if (realLinkJobs.length >= 3)
+        return realLinkJobs;
+    // Strategy 2: Repeated structured containers (custom career pages)
+    const structuredJobs = extractFromContainers(root, sourceUrl);
+    return structuredJobs.length > realLinkJobs.length
+        ? structuredJobs
+        : realLinkJobs;
+}
+function normalizeUrl(url) {
+    return url.replace(/\/+$/, "").replace(/\?.*$/, "");
+}
+// ---------------------------------------------------------------------------
+// Strategy 1: Link-based extraction
+// ---------------------------------------------------------------------------
+function extractFromLinks(root, sourceUrl) {
+    const jobs = [];
+    const seen = new Set();
+    const links = root.querySelectorAll("a[href]");
+    for (const link of links) {
+        const href = link.getAttribute("href");
+        if (!href)
+            continue;
+        if (!isJobLink(href))
+            continue;
+        const title = link.textContent.trim();
+        if (!title || title.length < 3 || title.length > 200)
+            continue;
+        let fullUrl;
+        try {
+            fullUrl = new URL(href, sourceUrl).href;
+        }
+        catch {
+            continue;
+        }
+        if (seen.has(fullUrl))
+            continue;
+        seen.add(fullUrl);
+        const id = createHash("sha256")
+            .update(`${fullUrl}:${title}`)
+            .digest("hex")
+            .slice(0, 12);
+        const parent = link.parentNode;
+        const parentText = parent?.textContent ?? "";
+        const location = extractLocation(parentText);
+        const department = findDepartmentHeading(link);
+        jobs.push({
+            id,
+            title,
+            company: new URL(sourceUrl).hostname.replace("www.", ""),
+            location,
+            workMode: location ? inferWorkMode(location) : null,
+            department,
+            url: fullUrl,
+            sourceUrl,
+            provider: "generic",
+            description: null,
+            postedAt: null,
+            extractedAt: new Date().toISOString(),
+            raw: null,
+        });
+    }
+    return jobs;
+}
+// ---------------------------------------------------------------------------
+// Strategy 2: Repeated container extraction
+// ---------------------------------------------------------------------------
+const JOB_CLASS_RE = /role|job|position|opening|career|listing|vacancy/i;
+function extractFromContainers(root, sourceUrl) {
+    const jobs = [];
+    const seen = new Set();
+    // Collect page headings — used to filter out section/department headers
+    const headings = new Set(root
+        .querySelectorAll("h1, h2, h3, h4, h5, h6")
+        .map((h) => h.textContent.trim().toLowerCase())
+        .filter((t) => t.length > 0));
+    // Group elements by their first CSS class when it matches job-related terms
+    const groups = new Map();
+    for (const el of root.querySelectorAll("[class]")) {
+        const cls = el.getAttribute("class") ?? "";
+        if (!JOB_CLASS_RE.test(cls))
+            continue;
+        const key = cls.split(/\s+/)[0];
+        const arr = groups.get(key);
+        if (arr)
+            arr.push(el);
+        else
+            groups.set(key, [el]);
+    }
+    // Pick the single best group: most cards after filtering.
+    // Processing multiple groups causes wrong departments when parent/child
+    // groups both qualify (the parent finds the PREVIOUS section's heading).
+    let bestCards = [];
+    for (const [, elements] of groups) {
+        if (elements.length < 3)
+            continue;
+        // Filter to "job card" sized containers: must have 2-6 non-SVG <p> tags.
+        // This excludes single-text sub-components (title/location wrappers)
+        // and large section wrappers that contain many jobs.
+        const cards = elements.filter((el) => {
+            const pCount = el
+                .querySelectorAll("p")
+                .filter((p) => !isInsideSvg(p)).length;
+            return pCount >= 2 && pCount <= 6;
+        });
+        if (cards.length > bestCards.length) {
+            bestCards = cards;
+        }
+    }
+    if (bestCards.length < 3)
+        return jobs;
+    for (const el of bestCards) {
+        const title = findTitle(el);
+        if (!title)
+            continue;
+        if (isLikelyLocation(title))
+            continue;
+        if (headings.has(title.toLowerCase()))
+            continue;
+        const id = createHash("sha256")
+            .update(`${sourceUrl}:${title}`)
+            .digest("hex")
+            .slice(0, 12);
+        if (seen.has(id))
+            continue;
+        seen.add(id);
+        const location = findLocationText(el);
+        const department = findDepartmentHeading(el);
+        jobs.push({
+            id,
+            title,
+            company: new URL(sourceUrl).hostname.replace("www.", ""),
+            location,
+            workMode: location ? inferWorkMode(location) : null,
+            department,
+            url: sourceUrl,
+            sourceUrl,
+            provider: "generic",
+            description: null,
+            postedAt: null,
+            extractedAt: new Date().toISOString(),
+            raw: null,
+        });
+    }
+    return jobs;
+}
+/**
+ * Reject strings that look like location/city text rather than job titles.
+ * E.g. "SF or NYC", "NYC or Remote (EST)", "SF, NYC, or Remote (USA)"
+ */
+function isLikelyLocation(text) {
+    const words = text.split(/[\s,()]+/).filter(Boolean);
+    if (words.length === 0)
+        return false;
+    const locationish = words.filter((w) => /^(or|and|[A-Z]{2,3}|remote|usa|us|est|pst|cst|mst)$/i.test(w));
+    return locationish.length === words.length;
+}
+/**
+ * Find the job title inside a container element.
+ * Looks for the first <p> or heading with meaningful short text,
+ * skipping SVG desc content and generic labels.
+ */
+function findTitle(container) {
+    const candidates = container.querySelectorAll("h1, h2, h3, h4, h5, h6, p");
+    for (const el of candidates) {
+        // Skip elements nested inside <svg>
+        if (isInsideSvg(el))
+            continue;
+        const text = el.textContent.trim();
+        if (text.length < 3 || text.length > 200)
+            continue;
+        // Skip generic labels
+        if (/^(location|department|team|type|apply|learn more|remote|hybrid|onsite|on-site)$/i.test(text))
+            continue;
+        return text;
+    }
+    return null;
+}
+/**
+ * Find location text inside a container element.
+ * Looks for the second <p> or an element whose class contains "location".
+ */
+function findLocationText(container) {
+    // Try class-based: element whose class contains "location"
+    for (const el of container.querySelectorAll("[class]")) {
+        const cls = el.getAttribute("class") ?? "";
+        if (!/location/i.test(cls))
+            continue;
+        if (isInsideSvg(el))
+            continue;
+        // Get text from <p> children, or direct text
+        const p = el.querySelector("p");
+        const text = (p ?? el).textContent.trim();
+        // Filter out SVG noise (e.g. "location" from <desc>)
+        const cleaned = text.replace(/^location\s*/i, "").trim();
+        if (cleaned.length >= 2 && cleaned.length <= 100)
+            return cleaned;
+    }
+    // Fallback: second <p> that isn't inside an SVG
+    const paragraphs = container.querySelectorAll("p");
+    let count = 0;
+    for (const p of paragraphs) {
+        if (isInsideSvg(p))
+            continue;
+        count++;
+        if (count === 2) {
+            const text = p.textContent.trim();
+            if (text.length >= 2 && text.length <= 100)
+                return text;
+        }
+    }
+    return null;
+}
+function isInsideSvg(el) {
+    let node = el;
+    while (node) {
+        if (node.rawTagName?.toLowerCase() === "svg")
+            return true;
+        node = node.parentNode;
+    }
+    return false;
+}
+function isJobLink(href) {
+    const lower = href.toLowerCase();
+    const patterns = [
+        "/jobs/",
+        "/job/",
+        "/careers/",
+        "/positions/",
+        "/position/",
+        "/openings/",
+        "/opening/",
+        "/apply/",
+        "/role/",
+        "/roles/",
+        "/vacancies/",
+        "/vacancy/",
+    ];
+    return patterns.some((p) => lower.includes(p));
+}
+function extractLocation(text) {
+    // Common location patterns in job listing context
+    const patterns = [
+        /(?:location|based in|located in)[:\s]+([^,\n]{3,50})/i,
+        /(?:remote|hybrid|onsite|on-site)/i,
+    ];
+    for (const pattern of patterns) {
+        const match = text.match(pattern);
+        if (match)
+            return match[1]?.trim() ?? match[0].trim();
+    }
+    return null;
+}
+function findDepartmentHeading(el) {
+    // Walk up the DOM, then backwards through siblings at each level.
+    // Check both siblings themselves and their descendants for headings
+    // (handles cases like <div class="info"><h3>Dept</h3></div>).
+    let node = el.parentNode;
+    for (let depth = 0; depth < 5 && node; depth++) {
+        let sibling = node.previousElementSibling;
+        for (let i = 0; i < 5 && sibling; i++) {
+            const tag = sibling.rawTagName?.toLowerCase();
+            if (tag && /^h[1-6]$/.test(tag)) {
+                return sibling.textContent.trim() || null;
+            }
+            const nested = sibling.querySelector("h1, h2, h3, h4, h5, h6");
+            if (nested) {
+                return nested.textContent.trim() || null;
+            }
+            sibling = sibling.previousElementSibling;
+        }
+        node = node.parentNode;
+    }
+    return null;
+}
+function inferWorkMode(location) {
+    const lower = location.toLowerCase();
+    if (lower.includes("remote"))
+        return "remote";
+    if (lower.includes("hybrid"))
+        return "hybrid";
+    return null;
+}

package/dist/core/providers/greenhouse.d.ts

@@ -0,0 +1,6 @@
+import type { Job, SearchCriteria } from "../../types/index.js";
+/**
+ * Fetch jobs from Greenhouse's public JSON API.
+ * No auth required. Returns all jobs as a single dump.
+ */
+export declare function extractGreenhouseJobs(boardToken: string, sourceUrl: string, _criteria: SearchCriteria, saveRaw?: boolean): Promise<Job[]>;

package/dist/core/providers/greenhouse.js

@@ -0,0 +1,47 @@
+import { createHash } from "node:crypto";
+import { saveRawResponse } from "../save-raw.js";
+/**
+ * Fetch jobs from Greenhouse's public JSON API.
+ * No auth required. Returns all jobs as a single dump.
+ */
+export async function extractGreenhouseJobs(boardToken, sourceUrl, _criteria, saveRaw = false) {
+    const apiUrl = `https://boards-api.greenhouse.io/v1/boards/${boardToken}/jobs`;
+    const response = await fetch(apiUrl);
+    if (!response.ok) {
+        throw new Error(`Greenhouse API returned ${response.status} for board "${boardToken}"`);
+    }
+    const data = (await response.json());
+    if (saveRaw)
+        await saveRawResponse("greenhouse", boardToken, data);
+    return data.jobs.map((gj) => {
+        const id = createHash("sha256")
+            .update(`${gj.absolute_url}:${gj.title}:${boardToken}`)
+            .digest("hex")
+            .slice(0, 12);
+        return {
+            id,
+            title: gj.title,
+            company: boardToken,
+            location: gj.location?.name ?? null,
+            workMode: inferWorkMode(gj.location?.name),
+            department: gj.departments?.[0]?.name ?? null,
+            url: gj.absolute_url,
+            sourceUrl,
+            provider: "greenhouse",
+            description: null,
+            postedAt: gj.updated_at ?? null,
+            extractedAt: new Date().toISOString(),
+            raw: gj,
+        };
+    });
+}
+function inferWorkMode(location) {
+    if (!location)
+        return null;
+    const lower = location.toLowerCase();
+    if (lower.includes("remote"))
+        return "remote";
+    if (lower.includes("hybrid"))
+        return "hybrid";
+    return null;
+}

package/dist/core/providers/lever.d.ts

@@ -0,0 +1,7 @@
+import type { Job, SearchCriteria } from "../../types/index.js";
+/**
+ * Fetch jobs from Lever's public postings API.
+ * No auth required, but many companies don't have it enabled (returns 404).
+ * Must fall through gracefully.
+ */
+export declare function extractLeverJobs(companySlug: string, sourceUrl: string, _criteria: SearchCriteria, saveRaw?: boolean): Promise<Job[]>;

package/dist/core/providers/lever.js

@@ -0,0 +1,60 @@
+import { createHash } from "node:crypto";
+import { saveRawResponse } from "../save-raw.js";
+/**
+ * Fetch jobs from Lever's public postings API.
+ * No auth required, but many companies don't have it enabled (returns 404).
+ * Must fall through gracefully.
+ */
+export async function extractLeverJobs(companySlug, sourceUrl, _criteria, saveRaw = false) {
+    const apiUrl = `https://api.lever.co/v0/postings/${companySlug}`;
+    const response = await fetch(apiUrl);
+    if (!response.ok) {
+        throw new Error(`Lever API returned ${response.status} for "${companySlug}"`);
+    }
+    const data = (await response.json());
+    if (!Array.isArray(data)) {
+        throw new Error(`Lever API returned unexpected format for "${companySlug}"`);
+    }
+    if (saveRaw)
+        await saveRawResponse("lever", companySlug, data);
+    return data.map((lp) => {
+        const id = createHash("sha256")
+            .update(`${lp.hostedUrl}:${lp.text}:${companySlug}`)
+            .digest("hex")
+            .slice(0, 12);
+        return {
+            id,
+            title: lp.text,
+            company: companySlug,
+            location: lp.categories?.location ?? null,
+            workMode: inferWorkMode(lp),
+            department: lp.categories?.department ?? lp.categories?.team ?? null,
+            url: lp.hostedUrl,
+            sourceUrl,
+            provider: "lever",
+            description: null,
+            postedAt: lp.createdAt ? new Date(lp.createdAt).toISOString() : null,
+            extractedAt: new Date().toISOString(),
+            raw: lp,
+        };
+    });
+}
+function inferWorkMode(posting) {
+    if (posting.workplaceType) {
+        const wt = posting.workplaceType.toLowerCase();
+        if (wt.includes("remote"))
+            return "remote";
+        if (wt.includes("hybrid"))
+            return "hybrid";
+        if (wt.includes("onsite") || wt.includes("on-site"))
+            return "onsite";
+    }
+    const loc = posting.categories?.location?.toLowerCase();
+    if (loc) {
+        if (loc.includes("remote"))
+            return "remote";
+        if (loc.includes("hybrid"))
+            return "hybrid";
+    }
+    return null;
+}

package/dist/core/providers/yc.d.ts

@@ -0,0 +1,7 @@
+import type { Job, SearchCriteria } from "../../types/index.js";
+/**
+ * Fetch jobs from YC's Work at a Startup via Algolia.
+ * Maps SearchCriteria to Algolia filters. Falls back to URL query params
+ * for backward compatibility with direct WaaS URLs.
+ */
+export declare function extractYcJobs(sourceUrl: string, criteria: SearchCriteria, saveRaw?: boolean): Promise<Job[]>;