jobcrawl 0.1.0

package/dist/core/providers/yc.js

@@ -0,0 +1,320 @@
+import { createHash } from "node:crypto";
+import { readFileSync, existsSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { saveRawResponse } from "../save-raw.js";
+const HITS_PER_PAGE = 100;
+const INDEX_BY_DATE = "WaaSPublicCompanyJob_created_at_desc_production";
+const INDEX_BY_RELEVANCE = "WaaSPublicCompanyJob_production";
+// --- Value mapping tables ---
+const ROLE_MAP = {
+    engineering: "eng",
+    design: "design",
+    product: "product",
+    science: "science",
+    sales: "sales",
+    marketing: "marketing",
+    support: "support",
+    operations: "operations",
+    "recruiting-hr": "recruiting",
+    finance: "finance",
+    legal: "legal",
+};
+const ENG_TYPE_MAP = {
+    android: "android",
+    backend: "be",
+    "data-science": "data_sci",
+    devops: "devops",
+    "embedded-systems": "embedded",
+    "engineering-manager": "eng_mgmt",
+    frontend: "fe",
+    "full-stack": "fs",
+    ios: "ios",
+    "machine-learning": "ml",
+    "qa-engineer": "qa",
+    robotics: "robotics",
+    hardware: "hw",
+    electrical: "electrical",
+    mechanical: "mechanical",
+    bioengineering: "bio",
+    "chemical-engineering": "chemical",
+};
+const DESIGN_TYPE_MAP = {
+    "web-design": "web",
+    "mobile-design": "mobile",
+    "product-design": "product",
+    "ui-ux": "ui_ux",
+    "user-research": "user_research",
+    "brand-graphic-design": "brand_graphic",
+    illustration: "illustration",
+    animation: "animation",
+    hardware: "hardware",
+    "3d-ar-vr": "ar_vr",
+    "design-manager": "design_mgmt",
+};
+const SCIENCE_TYPE_MAP = {
+    biology: "bio",
+    biotechnology: "biotech",
+    chemistry: "chem",
+    genetics: "genetics",
+    healthcare: "health",
+    immunology: "immuno",
+    laboratory: "lab",
+    oncology: "onc",
+    pharmacology: "pharma",
+    "process-engineer": "process",
+    research: "research",
+};
+const JOB_TYPE_MAP = {
+    fulltime: "fulltime",
+    internship: "intern",
+    contract: "contract",
+};
+const COMPANY_STAGE_MAP = {
+    seed: "seed",
+    "series-a": "series_a",
+    growth: "growth",
+    scale: "scale",
+};
+const WORK_MODE_MAP = {
+    remote: "only",
+    onsite: "no",
+    hybrid: "yes",
+};
+const ROLE_TYPE_FIELD_MAP = {
+    eng: "eng_type",
+    design: "design_type",
+    science: "science_type",
+};
+const ROLE_TYPE_VALUE_MAP = {
+    eng: ENG_TYPE_MAP,
+    design: DESIGN_TYPE_MAP,
+    science: SCIENCE_TYPE_MAP,
+};
+const COMPANY_SIZE_MAP = {
+    "1-10": "company_team_size <= 10",
+    "11-50": "company_team_size: 11 TO 50",
+    "51-300": "company_team_size: 51 TO 300",
+    "301+": "company_team_size >= 301",
+};
+// --- Filter helpers ---
+function mapValues(values, map) {
+    return values.map((v) => map[v] ?? v);
+}
+function orFilter(field, values, quoted = false) {
+    const parts = values.map((v) => quoted ? `${field}:"${v}"` : `${field}:${v}`);
+    return `(${parts.join(" OR ")})`;
+}
+function buildFilters(criteria, sourceUrl) {
+    const params = new URL(sourceUrl).searchParams;
+    const filters = [];
+    // role
+    const roles = criteria.role
+        ? mapValues(criteria.role, ROLE_MAP)
+        : fallbackArray(params, "role");
+    if (roles.length > 0)
+        filters.push(orFilter("role", roles));
+    // roleType → field depends on role
+    if (criteria.roleType && criteria.role) {
+        const algoliaRoles = mapValues(criteria.role, ROLE_MAP);
+        for (const algoliaRole of algoliaRoles) {
+            const field = ROLE_TYPE_FIELD_MAP[algoliaRole];
+            const valueMap = ROLE_TYPE_VALUE_MAP[algoliaRole];
+            if (field && valueMap) {
+                const mapped = mapValues(criteria.roleType, valueMap);
+                filters.push(orFilter(field, mapped));
+            }
+        }
+    }
+    else {
+        const roleTypeParam = params.get("role_type");
+        if (roleTypeParam && roleTypeParam !== "any")
+            filters.push(`(eng_type:${roleTypeParam})`);
+    }
+    // workMode → remote
+    const remoteModes = criteria.workMode
+        ? mapValues(criteria.workMode, WORK_MODE_MAP)
+        : fallbackArray(params, "remote");
+    if (remoteModes.length > 0)
+        filters.push(orFilter("remote", remoteModes));
+    // location
+    const location = criteria.location ?? params.get("locations");
+    if (location && location !== "any")
+        filters.push(`(locations_for_search:"${location}")`);
+    // jobType
+    const jobTypes = criteria.jobType
+        ? mapValues(criteria.jobType, JOB_TYPE_MAP)
+        : fallbackArray(params, "jobType");
+    if (jobTypes.length > 0)
+        filters.push(orFilter("job_type", jobTypes, true));
+    // minExperience
+    const minExps = criteria.minExperience
+        ? criteria.minExperience.map(String)
+        : fallbackArray(params, "minExperience");
+    if (minExps.length > 0)
+        filters.push(orFilter("min_experience", minExps));
+    // companyStage
+    const stages = criteria.companyStage
+        ? mapValues(criteria.companyStage, COMPANY_STAGE_MAP)
+        : fallbackArray(params, "companyStage");
+    if (stages.length > 0)
+        filters.push(orFilter("company_waas_stage", stages));
+    // industry
+    const industries = criteria.industry ?? fallbackQuotedArray(params, "industry");
+    if (industries.length > 0)
+        filters.push(orFilter("company_parent_sector", industries, true));
+    // companySize → range syntax
+    if (criteria.companySize && criteria.companySize.length > 0) {
+        const rangeClauses = criteria.companySize
+            .map((s) => COMPANY_SIZE_MAP[s])
+            .filter(Boolean);
+        if (rangeClauses.length > 0)
+            filters.push(`(${rangeClauses.join(" OR ")})`);
+    }
+    // hasSalary
+    if (criteria.hasSalary)
+        filters.push("(has_salary:true)");
+    // hasEquity
+    if (criteria.hasEquity)
+        filters.push("(has_equity:true)");
+    // hasInterviewProcess
+    if (criteria.hasInterviewProcess)
+        filters.push("(has_interview_process:true)");
+    // visaSponsorship
+    if (criteria.visaSponsorship)
+        filters.push("(us_visa_required:none OR us_visa_required:possible)");
+    return filters.join(" AND ");
+}
+function fallbackArray(params, key) {
+    const val = params.get(key);
+    if (!val || val === "any")
+        return [];
+    return [val];
+}
+function fallbackQuotedArray(params, key) {
+    const val = params.get(key);
+    if (!val || val === "any")
+        return [];
+    return [val];
+}
+// --- Algolia API ---
+async function fetchPage(appId, apiKey, query, filters, indexName, page) {
+    const algoliaParams = new URLSearchParams({
+        query,
+        page: String(page),
+        filters,
+        attributesToRetrieve: JSON.stringify(["*"]),
+        attributesToHighlight: JSON.stringify([]),
+        attributesToSnippet: JSON.stringify([]),
+        hitsPerPage: String(HITS_PER_PAGE),
+        clickAnalytics: "true",
+        distinct: "true",
+    });
+    const url = `https://${appId}-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=${encodeURIComponent("Algolia for JavaScript (3.35.1); Browser")}&x-algolia-application-id=${appId}&x-algolia-api-key=${apiKey}`;
+    const response = await fetch(url, {
+        method: "POST",
+        headers: {
+            accept: "application/json",
+            "content-type": "application/x-www-form-urlencoded",
+            Origin: "https://www.workatastartup.com",
+            Referer: "https://www.workatastartup.com/",
+        },
+        body: JSON.stringify({
+            requests: [
+                {
+                    indexName,
+                    params: algoliaParams.toString(),
+                },
+            ],
+        }),
+    });
+    if (!response.ok) {
+        throw new Error(`Algolia API returned ${response.status}`);
+    }
+    return (await response.json());
+}
+// --- Job mapping ---
+function mapHit(hit, sourceUrl) {
+    const id = createHash("sha256")
+        .update(`${hit.objectID}:${hit.title}:yc`)
+        .digest("hex")
+        .slice(0, 12);
+    const jobUrl = hit.search_path ??
+        `https://www.workatastartup.com/jobs/${hit.objectID}`;
+    return {
+        id,
+        title: hit.title,
+        company: hit.company_name,
+        location: hit.locations_for_search?.[0] ?? null,
+        workMode: inferWorkMode(hit.remote),
+        department: hit.role ?? null,
+        url: jobUrl,
+        sourceUrl,
+        provider: "yc",
+        description: hit.description?.slice(0, 200) ?? null,
+        postedAt: hit.created_at ?? null,
+        extractedAt: new Date().toISOString(),
+        raw: hit,
+    };
+}
+function inferWorkMode(remote) {
+    if (!remote)
+        return null;
+    const r = remote.toLowerCase();
+    if (r === "only" || r === "yes")
+        return "remote";
+    if (r === "no")
+        return null;
+    return null;
+}
+// --- Entry point ---
+/**
+ * Fetch jobs from YC's Work at a Startup via Algolia.
+ * Maps SearchCriteria to Algolia filters. Falls back to URL query params
+ * for backward compatibility with direct WaaS URLs.
+ */
+export async function extractYcJobs(sourceUrl, criteria, saveRaw = false) {
+    let appId = process.env.YC_ALGOLIA_APP_ID;
+    let apiKey = process.env.YC_ALGOLIA_API_KEY;
+    if (!appId || !apiKey) {
+        const credPath = join(homedir(), ".jobcrawl", "credentials.json");
+        if (existsSync(credPath)) {
+            const creds = JSON.parse(readFileSync(credPath, "utf-8"));
+            appId = appId || creds.yc?.algoliaAppId;
+            apiKey = apiKey || creds.yc?.algoliaApiKey;
+        }
+    }
+    if (!appId || !apiKey) {
+        throw new Error("YC provider requires credentials. Run `jobcrawl init` and add your Algolia keys to ~/.jobcrawl/credentials.json");
+    }
+    const queries = criteria.keywords.length > 0
+        ? criteria.keywords
+        : [""];
+    const filters = buildFilters(criteria, sourceUrl);
+    // Fetch each keyword as a separate Algolia query and deduplicate
+    const jobMap = new Map();
+    const allHits = [];
+    for (const query of queries) {
+        const indexName = query ? INDEX_BY_RELEVANCE : INDEX_BY_DATE;
+        const first = await fetchPage(appId, apiKey, query, filters, indexName, 0);
+        const result = first.results[0];
+        for (const hit of result.hits) {
+            if (!jobMap.has(hit.objectID)) {
+                jobMap.set(hit.objectID, mapHit(hit, sourceUrl));
+                allHits.push(hit);
+            }
+        }
+        for (let page = 1; page < result.nbPages; page++) {
+            const resp = await fetchPage(appId, apiKey, query, filters, indexName, page);
+            for (const hit of resp.results[0].hits) {
+                if (!jobMap.has(hit.objectID)) {
+                    jobMap.set(hit.objectID, mapHit(hit, sourceUrl));
+                    allHits.push(hit);
+                }
+            }
+        }
+    }
+    if (saveRaw)
+        await saveRawResponse("yc", "workatastartup", allHits);
+    return [...jobMap.values()];
+}

package/dist/core/resolve-iframe.d.ts

@@ -0,0 +1,6 @@
+import type { ProviderDetection } from "../types/index.js";
+/**
+ * When a career page embeds an ATS via iframe, extract the board token
+ * from the iframe src URL so we can call the ATS JSON API directly.
+ */
+export declare function resolveIframeToken(iframeSrc: string): ProviderDetection | null;

package/dist/core/resolve-iframe.js

@@ -0,0 +1,51 @@
+/**
+ * When a career page embeds an ATS via iframe, extract the board token
+ * from the iframe src URL so we can call the ATS JSON API directly.
+ */
+export function resolveIframeToken(iframeSrc) {
+    try {
+        const url = new URL(iframeSrc.startsWith("//") ? `https:${iframeSrc}` : iframeSrc);
+        // Greenhouse embed: boards.greenhouse.io/embed/job_board/js?for={token}
+        // Must check before general Greenhouse pattern to avoid extracting "embed" as token
+        if (url.pathname.includes("/embed/job_board")) {
+            const token = url.searchParams.get("for");
+            if (token)
+                return { provider: "greenhouse", boardToken: token };
+        }
+        // Greenhouse iframe: boards.greenhouse.io/{token} or job-boards.greenhouse.io/{token}
+        if (url.hostname === "boards.greenhouse.io" ||
+            url.hostname === "job-boards.greenhouse.io") {
+            const token = url.pathname.split("/").filter(Boolean)[0];
+            if (token)
+                return { provider: "greenhouse", boardToken: token };
+        }
+        // Lever iframe: jobs.lever.co/{company}
+        if (url.hostname === "jobs.lever.co") {
+            const company = url.pathname.split("/").filter(Boolean)[0];
+            if (company)
+                return { provider: "lever", boardToken: company };
+        }
+        // Ashby iframe: jobs.ashbyhq.com/{company}
+        if (url.hostname === "jobs.ashbyhq.com") {
+            const company = url.pathname.split("/").filter(Boolean)[0];
+            if (company)
+                return { provider: "ashby", boardToken: company };
+        }
+        // Workday iframe: {tenant}.wd{n}.myworkdayjobs.com
+        if (url.hostname.includes(".myworkdayjobs.com")) {
+            const tenant = url.hostname.split(".")[0];
+            if (tenant)
+                return { provider: "workday", boardToken: tenant };
+        }
+        // BambooHR iframe: {company}.bamboohr.com
+        if (url.hostname.includes(".bamboohr.com")) {
+            const company = url.hostname.split(".")[0];
+            if (company)
+                return { provider: "bamboohr", boardToken: company };
+        }
+    }
+    catch {
+        // Invalid URL
+    }
+    return null;
+}

package/dist/core/save-raw.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Save a raw API response to ~/.jobcrawl/raw/{provider}/{boardToken}_{timestamp}.json
+ */
+export declare function saveRawResponse(provider: string, boardToken: string, data: unknown): Promise<void>;

package/dist/core/save-raw.js

@@ -0,0 +1,13 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+/**
+ * Save a raw API response to ~/.jobcrawl/raw/{provider}/{boardToken}_{timestamp}.json
+ */
+export async function saveRawResponse(provider, boardToken, data) {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+    const dir = join(homedir(), ".jobcrawl", "raw", provider);
+    await mkdir(dir, { recursive: true });
+    const filename = `${boardToken}_${timestamp}.json`;
+    await writeFile(join(dir, filename), JSON.stringify(data, null, 2));
+}

package/dist/data/companies.d.ts

@@ -0,0 +1,9 @@
+import type { Provider } from "../types/index.js";
+export interface CompanyEntry {
+    company: string;
+    slug: string;
+    provider?: Provider;
+    fallback?: string;
+    section: string;
+}
+export declare const companies: CompanyEntry[];