jobcrawl 0.1.0

package/src/core/detect-provider.ts

@@ -0,0 +1,162 @@
+import type { ProviderDetection } from "../types/index.js";
+
+/**
+ * Detect ATS provider from HTML content and URL.
+ * Checks for iframes, embed scripts, and DOM signatures.
+ * Extracts board token/slug for API calls when possible.
+ */
+export function detectProvider(html: string, url: string): ProviderDetection {
+  // 1. URL pattern matching (direct ATS board URLs)
+  const urlMatch = detectFromUrl(url);
+  if (urlMatch.provider !== "unknown") return urlMatch;
+
+  // 2. HTML signature detection (embedded ATS on custom domains)
+  return detectFromHtml(html);
+}
+
+function detectFromUrl(url: string): ProviderDetection {
+  const u = new URL(url);
+
+  // Greenhouse: boards.greenhouse.io/{token} or job-boards.greenhouse.io/{token}
+  if (
+    u.hostname === "boards.greenhouse.io" ||
+    u.hostname === "job-boards.greenhouse.io"
+  ) {
+    const token = u.pathname.split("/").filter(Boolean)[0];
+    if (token) return { provider: "greenhouse", boardToken: token };
+  }
+
+  // Lever: jobs.lever.co/{company}
+  if (u.hostname === "jobs.lever.co") {
+    const company = u.pathname.split("/").filter(Boolean)[0];
+    if (company) return { provider: "lever", boardToken: company };
+  }
+
+  // Ashby: jobs.ashbyhq.com/{company}
+  if (u.hostname === "jobs.ashbyhq.com") {
+    const company = u.pathname.split("/").filter(Boolean)[0];
+    if (company) return { provider: "ashby", boardToken: company };
+  }
+
+  // Workday: {company}.wd{n}.myworkdayjobs.com
+  if (u.hostname.includes(".myworkdayjobs.com")) {
+    const tenant = u.hostname.split(".")[0];
+    if (tenant) return { provider: "workday", boardToken: tenant };
+  }
+
+  // BambooHR: {company}.bamboohr.com
+  if (u.hostname.includes(".bamboohr.com")) {
+    const company = u.hostname.split(".")[0];
+    if (company) return { provider: "bamboohr", boardToken: company };
+  }
+
+  return { provider: "unknown", boardToken: null };
+}
+
+function detectFromHtml(html: string): ProviderDetection {
+  // Greenhouse embed: <div id="grnhse_app"> or greenhouse embed script
+  const greenhouseIframe = html.match(
+    /iframe[^>]+src=["']([^"']*(?:boards|job-boards)\.greenhouse\.io[^"']*)/i
+  );
+  if (greenhouseIframe) {
+    const token = extractTokenFromUrl(
+      greenhouseIframe[1],
+      "boards.greenhouse.io"
+    );
+    if (token) return { provider: "greenhouse", boardToken: token };
+  }
+
+  const greenhouseEmbed = html.match(
+    /src=["'][^"']*api\.greenhouse\.io\/v1\/boards\/([^/"']+)/i
+  );
+  if (greenhouseEmbed) {
+    return { provider: "greenhouse", boardToken: greenhouseEmbed[1] };
+  }
+
+  if (html.includes('id="grnhse_app"') || html.includes("id='grnhse_app'")) {
+    // Look for the greenhouse board token in embed script
+    const tokenMatch = html.match(
+      /Grnhse\.Settings\.board_token\s*=\s*["']([^"']+)/
+    );
+    if (tokenMatch) {
+      return { provider: "greenhouse", boardToken: tokenMatch[1] };
+    }
+    const scriptSrc = html.match(
+      /src=["'][^"']*(?:boards|job-boards)\.greenhouse\.io\/embed\/job_board\/js\?for=([^"'&]+)/i
+    );
+    if (scriptSrc) {
+      return { provider: "greenhouse", boardToken: scriptSrc[1] };
+    }
+  }
+
+  // Lever embed
+  const leverIframe = html.match(
+    /iframe[^>]+src=["']([^"']*jobs\.lever\.co[^"']*)/i
+  );
+  if (leverIframe) {
+    const token = extractTokenFromUrl(leverIframe[1], "jobs.lever.co");
+    if (token) return { provider: "lever", boardToken: token };
+  }
+
+  if (html.includes("data-lever-")) {
+    const leverCompany = html.match(/data-lever-company=["']([^"']+)/i);
+    if (leverCompany) {
+      return { provider: "lever", boardToken: leverCompany[1] };
+    }
+  }
+
+  // Ashby embed
+  const ashbyIframe = html.match(
+    /iframe[^>]+src=["']([^"']*jobs\.ashbyhq\.com[^"']*)/i
+  );
+  if (ashbyIframe) {
+    const token = extractTokenFromUrl(ashbyIframe[1], "jobs.ashbyhq.com");
+    if (token) return { provider: "ashby", boardToken: token };
+  }
+
+  // Ashby embed script pattern
+  const ashbyScript = html.match(
+    /src=["'][^"']*ashbyhq\.com\/[^"']*["'][^>]*data-ashby-job-board-id=["']([^"']+)/i
+  );
+  if (ashbyScript) {
+    return { provider: "ashby", boardToken: ashbyScript[1] };
+  }
+
+  // Workday embed
+  const workdayIframe = html.match(
+    /iframe[^>]+src=["']([^"']*\.myworkdayjobs\.com[^"']*)/i
+  );
+  if (workdayIframe) {
+    const tenant = workdayIframe[1].match(/\/\/([^.]+)\.wd\d+\.myworkdayjobs/);
+    if (tenant) return { provider: "workday", boardToken: tenant[1] };
+  }
+
+  // BambooHR embed
+  const bambooIframe = html.match(
+    /iframe[^>]+src=["']([^"']*\.bamboohr\.com[^"']*)/i
+  );
+  if (bambooIframe) {
+    const company = bambooIframe[1].match(/\/\/([^.]+)\.bamboohr\.com/);
+    if (company) return { provider: "bamboohr", boardToken: company[1] };
+  }
+
+  return { provider: "unknown", boardToken: null };
+}
+
+function extractTokenFromUrl(
+  iframeSrc: string,
+  hostname: string
+): string | null {
+  try {
+    const url = new URL(
+      iframeSrc.startsWith("//") ? `https:${iframeSrc}` : iframeSrc
+    );
+    if (url.hostname === hostname) {
+      const parts = url.pathname.split("/").filter(Boolean);
+      return parts[0] || null;
+    }
+  } catch {
+    // Invalid URL
+  }
+  return null;
+}

package/src/core/discover-careers.ts

@@ -0,0 +1,117 @@
+import {
+  webSearch,
+  guessCareerUrls,
+  type SearchResult,
+} from "../utils/google-search.js";
+import { askClaude } from "../utils/llm.js";
+import { probePage } from "./fetch-page.js";
+
+export interface DiscoverResult {
+  company: string;
+  url: string | null;
+  verified: boolean;
+  error: string | null;
+}
+
+/**
+ * Discover the career page URL for a company.
+ *
+ * Strategy (in order):
+ * 1. Web search (DDG) + LLM to pick the best URL
+ * 2. Heuristic URL guessing ({company}.com/careers) + HTTP HEAD check
+ */
+export async function discoverCareerPage(
+  company: string
+): Promise<DiscoverResult> {
+  try {
+    // Strategy 1: Web search + LLM
+    const results = await webSearch(`"${company}" careers jobs`);
+
+    if (results.length > 0) {
+      const url = await pickCareerUrl(company, results);
+      if (url) {
+        return { company, url, verified: false, error: null };
+      }
+    }
+
+    // Strategy 2: Heuristic URL guessing
+    const guessed = await guessCareerUrls(company);
+    if (guessed.length > 0) {
+      return {
+        company,
+        url: guessed[0].url,
+        verified: false,
+        error: null,
+      };
+    }
+
+    return {
+      company,
+      url: null,
+      verified: false,
+      error: "Could not find career page",
+    };
+  } catch (err) {
+    return {
+      company,
+      url: null,
+      verified: false,
+      error: err instanceof Error ? err.message : String(err),
+    };
+  }
+}
+
+/**
+ * Verify that a discovered URL is a real career page.
+ */
+export async function verifyCareerPage(url: string): Promise<boolean> {
+  try {
+    const { html } = await probePage(url);
+    const lower = html.toLowerCase();
+
+    const signals = [
+      "careers",
+      "jobs",
+      "openings",
+      "positions",
+      "join our team",
+      "join us",
+      "we're hiring",
+      "open roles",
+      "apply now",
+      "job-boards.greenhouse.io",
+      "boards.greenhouse.io",
+      "jobs.lever.co",
+      "jobs.ashbyhq.com",
+      "grnhse_app",
+    ];
+
+    const matchCount = signals.filter((s) => lower.includes(s)).length;
+    return matchCount >= 2;
+  } catch {
+    return false;
+  }
+}
+
+async function pickCareerUrl(
+  company: string,
+  results: SearchResult[]
+): Promise<string | null> {
+  const formatted = results
+    .map((r, i) => `${i + 1}. ${r.title}\n   URL: ${r.url}\n   ${r.snippet}`)
+    .join("\n\n");
+
+  const prompt = `Given these search results for "${company} careers", return ONLY the URL of their official careers/jobs page. Not individual job postings or third-party job boards (Indeed, LinkedIn, Glassdoor).
+
+Search results:
+${formatted}
+
+Respond with ONLY the URL, nothing else. If you can't determine it, respond with "null".`;
+
+  const response = await askClaude(prompt);
+
+  const urlMatch = response.match(/https?:\/\/[^\s"'<>]+/);
+  if (urlMatch) return urlMatch[0];
+
+  return null;
+}

package/src/core/extract-jobs.ts

@@ -0,0 +1,50 @@
+import type { Job, Provider, Aggregator, SearchCriteria } from "../types/index.js";
+import { extractGreenhouseJobs } from "./providers/greenhouse.js";
+import { extractAshbyJobs } from "./providers/ashby.js";
+import { extractLeverJobs } from "./providers/lever.js";
+import { extractYcJobs } from "./aggregators/yc.js";
+import { extractGenericJobs } from "./providers/generic.js";
+
+/**
+ * Extract jobs via ATS JSON API. Used when provider + boardToken are known.
+ * Criteria is passed through so providers can apply server-side filtering.
+ */
+export async function extractViaApi(
+  provider: Provider,
+  boardToken: string,
+  sourceUrl: string,
+  criteria: SearchCriteria,
+  saveRaw = false
+): Promise<Job[]> {
+  switch (provider) {
+    case "greenhouse":
+      return extractGreenhouseJobs(boardToken, sourceUrl, criteria, saveRaw);
+    case "ashby":
+      return extractAshbyJobs(boardToken, sourceUrl, criteria, saveRaw);
+    case "lever":
+      return extractLeverJobs(boardToken, sourceUrl, criteria, saveRaw);
+    default:
+      throw new Error(`No API extractor for provider "${provider}"`);
+  }
+}
+
+/**
+ * Extract jobs via aggregator. Aggregators search across multiple companies.
+ */
+export async function extractViaAggregator(
+  aggregator: Aggregator,
+  criteria: SearchCriteria,
+  saveRaw = false
+): Promise<Job[]> {
+  switch (aggregator) {
+    case "yc":
+      return extractYcJobs("https://workatastartup.com", criteria, saveRaw);
+  }
+}
+
+/**
+ * Extract jobs from rendered HTML. Used when no ATS API is available.
+ */
+export function extractFromHtml(html: string, sourceUrl: string): Job[] {
+  return extractGenericJobs(html, sourceUrl);
+}

package/src/core/fetch-page.ts

@@ -0,0 +1,41 @@
+import { BrowserSession, BrowserNotAvailableError } from "./browser.js";
+import type { FetchResult } from "../types/index.js";
+
+/**
+ * Quick HTTP probe — simple GET, no browser.
+ * Enough to detect ATS signals in static HTML.
+ */
+export async function probePage(url: string): Promise<FetchResult> {
+  const response = await fetch(url, {
+    headers: {
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    },
+    redirect: "follow",
+  });
+  const html = await response.text();
+  return { html, finalUrl: response.url };
+}
+
+/**
+ * Full JS rendering via agent-browser.
+ * Falls back to probePage when agent-browser is not installed.
+ */
+export async function renderPage(url: string): Promise<FetchResult> {
+  const session = new BrowserSession();
+  try {
+    await session.open(url);
+    const [html, finalUrl] = await Promise.all([
+      session.getHtml(),
+      session.getUrl(),
+    ]);
+    return { html, finalUrl };
+  } catch (err) {
+    if (err instanceof BrowserNotAvailableError) {
+      return probePage(url);
+    }
+    throw err;
+  } finally {
+    await session.close();
+  }
+}

package/src/core/format-output.ts

@@ -0,0 +1,80 @@
+import type { Job, OutputFormat } from "../types/index.js";
+
+export function formatOutput(jobs: Job[], format: OutputFormat): string {
+  switch (format) {
+    case "json":
+      return JSON.stringify(jobs, null, 2);
+
+    case "table":
+      return formatTable(jobs);
+
+    case "csv":
+      return formatCsv(jobs);
+
+    case "markdown":
+      return formatMarkdown(jobs);
+  }
+}
+
+function formatTable(jobs: Job[]): string {
+  if (jobs.length === 0) return "No matching jobs found.";
+
+  const headers = ["Company", "Title", "Location", "Department", "URL"];
+  const rows = jobs.map((j) => [
+    j.company,
+    j.title,
+    j.location ?? "-",
+    j.department ?? "-",
+    j.url,
+  ]);
+
+  // Calculate column widths
+  const widths = headers.map((h, i) =>
+    Math.min(50, Math.max(h.length, ...rows.map((r) => r[i].length)))
+  );
+
+  const separator = widths.map((w) => "-".repeat(w)).join(" | ");
+  const headerLine = headers.map((h, i) => h.padEnd(widths[i])).join(" | ");
+  const bodyLines = rows.map((row) =>
+    row.map((cell, i) => cell.slice(0, widths[i]).padEnd(widths[i])).join(" | ")
+  );
+
+  return [headerLine, separator, ...bodyLines].join("\n");
+}
+
+function formatCsv(jobs: Job[]): string {
+  const headers = "company,title,location,department,workMode,url,postedAt";
+  const rows = jobs.map((j) =>
+    [
+      csvEscape(j.company),
+      csvEscape(j.title),
+      csvEscape(j.location ?? ""),
+      csvEscape(j.department ?? ""),
+      csvEscape(j.workMode ?? ""),
+      csvEscape(j.url),
+      csvEscape(j.postedAt ?? ""),
+    ].join(",")
+  );
+  return [headers, ...rows].join("\n");
+}
+
+function csvEscape(value: string): string {
+  if (value.includes(",") || value.includes('"') || value.includes("\n")) {
+    return `"${value.replace(/"/g, '""')}"`;
+  }
+  return value;
+}
+
+function formatMarkdown(jobs: Job[]): string {
+  if (jobs.length === 0) return "No matching jobs found.";
+
+  const lines = [
+    "| Company | Title | Location | Department |",
+    "| --- | --- | --- | --- |",
+    ...jobs.map(
+      (j) =>
+        `| ${j.company} | [${j.title}](${j.url}) | ${j.location ?? "-"} | ${j.department ?? "-"} |`
+    ),
+  ];
+  return lines.join("\n");
+}

package/src/core/match-jobs.ts

@@ -0,0 +1,56 @@
+import type { Job, SearchCriteria } from "../types/index.js";
+
+/**
+ * Filter jobs against search criteria. Case-insensitive substring matching.
+ * Jobs with null fields are included (don't exclude for missing data).
+ */
+export function matchJobs(jobs: Job[], criteria: SearchCriteria): Job[] {
+  return jobs.filter((job) => {
+    // Keywords: any keyword must match title (include if no keywords specified)
+    if (criteria.keywords.length > 0) {
+      const title = job.title.toLowerCase();
+      const hasMatch = criteria.keywords.some((kw) =>
+        title.includes(kw.toLowerCase())
+      );
+      if (!hasMatch) return false;
+    }
+
+    // Exclude keywords: exclude if any match title
+    if (criteria.excludeKeywords.length > 0) {
+      const title = job.title.toLowerCase();
+      const hasExclude = criteria.excludeKeywords.some((kw) =>
+        title.includes(kw.toLowerCase())
+      );
+      if (hasExclude) return false;
+    }
+
+    // Location: substring match (include if job has no location)
+    if (criteria.location && job.location) {
+      if (
+        !job.location.toLowerCase().includes(criteria.location.toLowerCase())
+      ) {
+        return false;
+      }
+    }
+
+    // Work mode: include if job's workMode is in allowed set (include if null)
+    if (criteria.workMode && criteria.workMode.length > 0 && job.workMode) {
+      if (!criteria.workMode.includes(job.workMode)) return false;
+    }
+
+    // Departments: substring match (include if null)
+    if (
+      criteria.departments &&
+      criteria.departments.length > 0 &&
+      job.department
+    ) {
+      const dept = job.department.toLowerCase();
+      const hasMatch = criteria.departments.some((d) =>
+        dept.includes(d.toLowerCase())
+      );
+      if (!hasMatch) return false;
+    }
+
+    return true;
+  });
+}

package/src/core/providers/ashby.ts

@@ -0,0 +1,84 @@
+import { createHash } from "node:crypto";
+import type { Job, SearchCriteria } from "../../types/index.js";
+import { saveRawResponse } from "../save-raw.js";
+
+interface AshbyJob {
+  id: string;
+  title: string;
+  department: string | null;
+  team: string | null;
+  employmentType: string | null;
+  location: string | null;
+  isRemote: boolean | null;
+  workplaceType: string | null;
+  jobUrl: string;
+  publishedAt: string | null;
+  descriptionPlain?: string;
+}
+
+interface AshbyResponse {
+  jobs: AshbyJob[];
+}
+
+/**
+ * Fetch jobs from Ashby's public posting API.
+ * No auth required. Returns all jobs as a single dump.
+ */
+export async function extractAshbyJobs(
+  boardToken: string,
+  sourceUrl: string,
+  _criteria: SearchCriteria,
+  saveRaw = false
+): Promise<Job[]> {
+  const apiUrl = `https://api.ashbyhq.com/posting-api/job-board/${boardToken}`;
+  const response = await fetch(apiUrl);
+
+  if (!response.ok) {
+    throw new Error(
+      `Ashby API returned ${response.status} for board "${boardToken}"`
+    );
+  }
+
+  const data = (await response.json()) as AshbyResponse;
+
+  if (saveRaw) await saveRawResponse("ashby", boardToken, data);
+
+  return data.jobs.map((aj): Job => {
+    const id = createHash("sha256")
+      .update(`${aj.jobUrl}:${aj.title}:${boardToken}`)
+      .digest("hex")
+      .slice(0, 12);
+
+    return {
+      id,
+      title: aj.title,
+      company: boardToken,
+      location: aj.location ?? null,
+      workMode: inferWorkMode(aj),
+      department: aj.department ?? aj.team ?? null,
+      url: aj.jobUrl,
+      sourceUrl,
+      provider: "ashby",
+      description: aj.descriptionPlain?.slice(0, 200) ?? null,
+      postedAt: aj.publishedAt ?? null,
+      extractedAt: new Date().toISOString(),
+      raw: aj as unknown as Record<string, unknown>,
+    };
+  });
+}
+
+function inferWorkMode(job: AshbyJob): "remote" | "onsite" | "hybrid" | null {
+  if (job.isRemote === true) return "remote";
+  if (job.workplaceType) {
+    const wt = job.workplaceType.toLowerCase();
+    if (wt.includes("remote")) return "remote";
+    if (wt.includes("hybrid")) return "hybrid";
+    if (wt.includes("onsite") || wt.includes("on-site")) return "onsite";
+  }
+  if (job.location) {
+    const loc = job.location.toLowerCase();
+    if (loc.includes("remote")) return "remote";
+    if (loc.includes("hybrid")) return "hybrid";
+  }
+  return null;
+}