jobcrawl 0.1.0

package/src/entrypoints/cli/crawl.ts

@@ -0,0 +1,163 @@
+import { existsSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { bus } from "../../events.js";
+import { crawlAll } from "../../orchestrators/crawl-all.js";
+import { loadConfig, parseUrlList } from "../../utils/config.js";
+import { formatOutput } from "../../core/format-output.js";
+import { CONFIG_FILE } from "./init.js";
+import type { Config } from "../../utils/config.js";
+import type {
+  Target,
+  Aggregator,
+  SearchCriteria,
+  OutputFormat,
+} from "../../types/index.js";
+
+interface CrawlOptions {
+  urls?: string[];
+  file?: string;
+  aggregators?: string[];
+  keywords?: string[];
+  exclude?: string[];
+  location?: string;
+  remote?: boolean;
+  onsite?: boolean;
+  hybrid?: boolean;
+  department?: string[];
+  role?: string[];
+  roleType?: string[];
+  jobType?: string[];
+  minExperience?: string[];
+  companyStage?: string[];
+  industry?: string[];
+  companySize?: string[];
+  hasSalary?: boolean;
+  hasEquity?: boolean;
+  hasInterviewProcess?: boolean;
+  visaSponsorship?: boolean;
+  output?: OutputFormat;
+  out?: string;
+  concurrency?: string;
+  saveRaw?: boolean;
+  networkTimeout?: string;
+  maxBubbleLevels?: string;
+}
+
+export async function crawlCommand(opts: CrawlOptions): Promise<void> {
+  const { targets, config } = await resolveTargets(opts);
+
+  if (targets.length === 0) {
+    throw new Error(
+      "No targets provided. Use --urls, --file, or run `jobcrawl init` to set up default targets."
+    );
+  }
+
+  const criteria: SearchCriteria = {
+    keywords: opts.keywords ?? [],
+    excludeKeywords: opts.exclude ?? [],
+    location: opts.location ?? null,
+    workMode: buildWorkMode(opts),
+    departments: opts.department ?? null,
+    role: opts.role ?? null,
+    roleType: opts.roleType ?? null,
+    jobType: opts.jobType ?? null,
+    minExperience: opts.minExperience
+      ? opts.minExperience.map((v) => parseInt(v, 10))
+      : null,
+    companyStage: opts.companyStage ?? null,
+    industry: opts.industry ?? null,
+    companySize: opts.companySize ?? null,
+    hasSalary: opts.hasSalary ?? null,
+    hasEquity: opts.hasEquity ?? null,
+    hasInterviewProcess: opts.hasInterviewProcess ?? null,
+    visaSponsorship: opts.visaSponsorship ?? null,
+  };
+
+  const concurrency = opts.concurrency ? parseInt(opts.concurrency, 10) : 5;
+  const format = opts.output ?? "json";
+
+  // Parse --aggregators flag
+  const aggregators: Aggregator[] | undefined = opts.aggregators?.map((name) => {
+    if (name === "yc") return name;
+    throw new Error(`Unknown aggregator: "${name}". Available: yc`);
+  });
+
+  // Browser options: CLI flags override config defaults
+  const configBrowser = config?.defaults?.browser;
+  const browser = {
+    networkTimeout: opts.networkTimeout
+      ? parseInt(opts.networkTimeout, 10)
+      : configBrowser?.networkTimeout,
+    maxBubbleLevels: opts.maxBubbleLevels
+      ? parseInt(opts.maxBubbleLevels, 10)
+      : configBrowser?.maxBubbleLevels,
+  };
+
+  const result = await crawlAll(targets, criteria, {
+    concurrency,
+    saveRaw: opts.saveRaw,
+    browser,
+  }, aggregators);
+
+  const output = formatOutput(result.jobs, format);
+
+  if (opts.out) {
+    const { writeFile } = await import("node:fs/promises");
+    await writeFile(opts.out, output + "\n");
+    bus.emit("output:message", {
+      text: `Wrote ${result.jobs.length} jobs to ${opts.out}`,
+      style: "success",
+    });
+  } else {
+    bus.emit("output:text", { text: output });
+  }
+}
+
+async function resolveTargets(
+  opts: CrawlOptions
+): Promise<{ targets: Target[]; config: Config | null }> {
+  const targets: Target[] = [];
+  let config: Config | null = null;
+
+  // From --urls flag
+  if (opts.urls) {
+    for (const url of opts.urls) {
+      targets.push({ url });
+    }
+  }
+
+  // From --file flag
+  if (opts.file) {
+    config = await loadConfig(opts.file);
+    targets.push(...config.companies);
+  }
+
+  // From ~/.jobcrawl/config.yaml (default config)
+  if (targets.length === 0 && existsSync(CONFIG_FILE)) {
+    config = await loadConfig(CONFIG_FILE);
+    targets.push(...config.companies);
+  }
+
+  // Always load config for defaults even when using --urls
+  if (!config && existsSync(CONFIG_FILE)) {
+    config = await loadConfig(CONFIG_FILE);
+  }
+
+  // From stdin (if not a TTY and no other input)
+  if (targets.length === 0 && !process.stdin.isTTY) {
+    const input = await readFile("/dev/stdin", "utf-8");
+    targets.push(...parseUrlList(input));
+  }
+
+  return { targets, config };
+}
+
+function buildWorkMode(
+  opts: CrawlOptions
+): ("remote" | "onsite" | "hybrid")[] | null {
+  const modes: ("remote" | "onsite" | "hybrid")[] = [];
+  if (opts.remote) modes.push("remote");
+  if (opts.onsite) modes.push("onsite");
+  if (opts.hybrid) modes.push("hybrid");
+  return modes.length > 0 ? modes : null;
+}

package/src/entrypoints/cli/discover.ts

@@ -0,0 +1,96 @@
+import { readFile } from "node:fs/promises";
+import { bus } from "../../events.js";
+import { discoverAll } from "../../orchestrators/discover-all.js";
+import type { UrlTarget } from "../../types/index.js";
+import yaml from "js-yaml";
+
+interface DiscoverOptions {
+  companies?: string[];
+  file?: string;
+  output?: "urls" | "yaml" | "json";
+  out?: string;
+  verify?: boolean;
+  concurrency?: string;
+}
+
+export async function discoverCommand(opts: DiscoverOptions): Promise<void> {
+  const companies = await resolveCompanies(opts);
+
+  if (companies.length === 0) {
+    throw new Error("No company names provided. Use --companies or --file.");
+  }
+
+  const concurrency = opts.concurrency ? parseInt(opts.concurrency, 10) : 3;
+  const format = opts.output ?? "urls";
+
+  const result = await discoverAll(companies, {
+    concurrency,
+    verify: opts.verify ?? false,
+  });
+
+  const found = result.results.filter((r) => r.url !== null);
+  const output = formatDiscoverOutput(found, format);
+
+  if (opts.out) {
+    const { writeFile } = await import("node:fs/promises");
+    await writeFile(opts.out, output + "\n");
+    bus.emit("output:message", {
+      text: `Wrote ${found.length} targets to ${opts.out}`,
+      style: "success",
+    });
+  } else {
+    bus.emit("output:text", { text: output });
+  }
+}
+
+async function resolveCompanies(opts: DiscoverOptions): Promise<string[]> {
+  const companies: string[] = [];
+
+  if (opts.companies) {
+    companies.push(...opts.companies);
+  }
+
+  if (opts.file) {
+    const content = await readFile(opts.file, "utf-8");
+    const lines = content
+      .split("\n")
+      .map((l) => l.trim())
+      .filter((l) => l.length > 0 && !l.startsWith("#"));
+    companies.push(...lines);
+  }
+
+  // stdin if no other input and not a TTY
+  if (companies.length === 0 && !process.stdin.isTTY) {
+    const input = await readFile("/dev/stdin", "utf-8");
+    const lines = input
+      .split("\n")
+      .map((l) => l.trim())
+      .filter((l) => l.length > 0 && !l.startsWith("#"));
+    companies.push(...lines);
+  }
+
+  return companies;
+}
+
+function formatDiscoverOutput(
+  found: Array<{ company: string; url: string | null }>,
+  format: "urls" | "yaml" | "json"
+): string {
+  const targets: UrlTarget[] = found
+    .filter((r): r is { company: string; url: string } => r.url !== null)
+    .map((r) => ({ url: r.url, company: r.company }));
+
+  switch (format) {
+    case "urls":
+      return targets.map((t) => t.url).join("\n");
+
+    case "json":
+      return JSON.stringify(targets, null, 2);
+
+    case "yaml":
+      return yaml.dump({
+        targets: targets.map((t) => ({ url: t.url, company: t.company })),
+        defaults: { concurrency: 5 },
+      });
+  }
+}

package/src/entrypoints/cli/index.ts

@@ -0,0 +1,252 @@
+#!/usr/bin/env node
+
+import React from "react";
+import { render } from "ink";
+import { Command } from "commander";
+import { bus } from "../../events.js";
+import { mountPlain } from "./plain.js";
+import { App } from "./app.js";
+import { crawlUrlCommand } from "./crawl-url.js";
+import { crawlCommand } from "./crawl.js";
+import { crawlAggregatorsCommand } from "./crawl-aggregators.js";
+import { discoverCommand } from "./discover.js";
+import { initCommand } from "./init.js";
+
+/**
+ * Lifecycle manager. Mounts Ink (TTY) or plain text (piped) subscriber,
+ * executes the command, and handles teardown.
+ */
+function wrap(
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  fn: (...args: any[]) => Promise<void>
+): (...args: unknown[]) => void {
+  return (...args: unknown[]) => {
+    if (process.stdout.isTTY) {
+      // Interactive: mount Ink (handles run + done + cleanup internally)
+      const run = () => fn(...args);
+      const instance = render(React.createElement(App, { run }));
+      const onDone = () => {
+        bus.off("done", onDone);
+        setTimeout(() => instance.unmount(), 32);
+      };
+      bus.on("done", onDone);
+    } else {
+      // Piped: plain text subscriber
+      const cleanup = mountPlain();
+
+      fn(...args)
+        .then(() => {
+          bus.emit("done", { exitCode: 0 });
+        })
+        .catch((err: unknown) => {
+          bus.emit("error", {
+            message: err instanceof Error ? err.message : String(err),
+          });
+          bus.emit("done", { exitCode: 1 });
+        })
+        .finally(() => {
+          cleanup();
+        });
+    }
+  };
+}
+
+function addFilterOptions(cmd: Command): Command {
+  return cmd
+    .option("--role <roles...>", "Role filter (e.g. engineering, design)")
+    .option("--role-type <types...>", "Role sub-type (e.g. backend, frontend)")
+    .option("--job-type <types...>", "Job type (e.g. fulltime, internship)")
+    .option("--min-experience <years...>", "Min years experience (e.g. 0, 3)")
+    .option("--company-stage <stages...>", "Company stage (e.g. seed, growth)")
+    .option("--industry <industries...>", "Industry filter")
+    .option("--company-size <sizes...>", "Company size (e.g. 1-10, 11-50)")
+    .option("--has-salary", "Only jobs with salary listed")
+    .option("--has-equity", "Only jobs with equity")
+    .option("--has-interview-process", "Only jobs with interview process")
+    .option("--visa-sponsorship", "Only jobs not requiring US visa");
+}
+
+const program = new Command("jobcrawl")
+  .version("1.0.0")
+  .description("Crawl career pages for jobs matching your search criteria");
+
+addFilterOptions(
+  program
+    .command("crawl-url <url>")
+    .description("Crawl a single career page URL and return matching jobs")
+    .option("--keywords <terms...>", "Job title keywords to match")
+    .option("--exclude <terms...>", "Keywords to exclude")
+    .option("--location <location>", "Location filter")
+    .option("--remote", "Only remote jobs")
+    .option("--onsite", "Only onsite jobs")
+    .option("--hybrid", "Only hybrid jobs")
+    .option("--department <depts...>", "Department filter")
+    .option(
+      "--output <format>",
+      "Output format: json, table, markdown, csv",
+      "json"
+    )
+    .option("-o, --out <file>", "Write output to file")
+    .option("--save-raw", "Save raw API responses to ~/.jobcrawl/raw/")
+).action(
+    wrap(async (url: string, opts: Record<string, unknown>) => {
+      await crawlUrlCommand(url as string, opts);
+    })
+  );
+
+addFilterOptions(
+  program
+    .command("crawl")
+    .description("Crawl multiple career pages and return matching jobs")
+    .option("--urls <urls...>", "Career page URLs to crawl")
+    .option("--file <path>", "Config file with targets (YAML/JSON)")
+    .option("--keywords <terms...>", "Job title keywords to match")
+    .option("--exclude <terms...>", "Keywords to exclude")
+    .option("--location <location>", "Location filter")
+    .option("--remote", "Only remote jobs")
+    .option("--onsite", "Only onsite jobs")
+    .option("--hybrid", "Only hybrid jobs")
+    .option("--department <depts...>", "Department filter")
+    .option(
+      "--output <format>",
+      "Output format: json, table, markdown, csv",
+      "json"
+    )
+    .option("-o, --out <file>", "Write output to file")
+    .option("--concurrency <n>", "Max concurrent crawls", "5")
+    .option("--save-raw", "Save raw API responses to ~/.jobcrawl/raw/")
+    .option("--aggregators <names...>", "Also run aggregators (e.g. yc)")
+    .option("--network-timeout <ms>", "Timeout for browser network commands (ms)")
+    .option("--max-bubble-levels <n>", "Max parent levels to try when clicking job cards")
+).action(
+    wrap(async (opts: Record<string, unknown>) => {
+      await crawlCommand(opts);
+    })
+  );
+
+addFilterOptions(
+  program
+    .command("crawl-aggregators <aggregators...>")
+    .description("Crawl aggregator sources (e.g. yc) for matching jobs")
+    .option("--keywords <terms...>", "Job title keywords to match")
+    .option("--exclude <terms...>", "Keywords to exclude")
+    .option("--location <location>", "Location filter")
+    .option("--remote", "Only remote jobs")
+    .option("--onsite", "Only onsite jobs")
+    .option("--hybrid", "Only hybrid jobs")
+    .option("--department <depts...>", "Department filter")
+    .option(
+      "--output <format>",
+      "Output format: json, table, markdown, csv",
+      "json"
+    )
+    .option("-o, --out <file>", "Write output to file")
+    .option("--save-raw", "Save raw API responses to ~/.jobcrawl/raw/")
+).action(
+    wrap(async (aggregators: string[], opts: Record<string, unknown>) => {
+      await crawlAggregatorsCommand(aggregators, opts);
+    })
+  );
+
+program
+  .command("discover")
+  .description("Find career page URLs from company names")
+  .option("--companies <names...>", "Company names to search for")
+  .option("--file <path>", "File with company names (one per line)")
+  .option("--output <format>", "Output format: urls, yaml, json", "urls")
+  .option("-o, --out <file>", "Write output to file")
+  .option("--verify", "Verify each discovered URL is a real career page")
+  .option("--concurrency <n>", "Max concurrent searches", "3")
+  .action(
+    wrap(async (opts: Record<string, unknown>) => {
+      await discoverCommand(opts);
+    })
+  );
+
+program
+  .command("init")
+  .description("Create config file at ~/.jobcrawl/config.yaml")
+  .option("--force", "Overwrite existing config")
+  .action(
+    wrap(async (opts: Record<string, unknown>) => {
+      await initCommand(opts);
+    })
+  );
+
+program
+  .command("detect <url>")
+  .description("Detect which ATS provider a career page uses")
+  .action(
+    wrap(async (url: string) => {
+      const { probePage } = await import("../../core/fetch-page.js");
+      const { detectProvider } = await import("../../core/detect-provider.js");
+      const { html, finalUrl } = await probePage(url);
+      const result = detectProvider(html, finalUrl);
+      bus.emit("output:json", { data: { url, finalUrl, ...result } });
+    })
+  );
+
+addFilterOptions(
+  program
+    .command("match <file>")
+    .description("Filter a jobs JSON file against search criteria")
+    .option("--keywords <terms...>", "Job title keywords to match")
+    .option("--exclude <terms...>", "Keywords to exclude")
+    .option("--location <location>", "Location filter")
+    .option("--remote", "Only remote jobs")
+    .option("--onsite", "Only onsite jobs")
+    .option("--hybrid", "Only hybrid jobs")
+    .option("--department <depts...>", "Department filter")
+    .option(
+      "--output <format>",
+      "Output format: json, table, markdown, csv",
+      "json"
+    )
+).action(
+    wrap(async (file: string, opts: Record<string, unknown>) => {
+      const { readFile } = await import("node:fs/promises");
+      const { matchJobs } = await import("../../core/match-jobs.js");
+      const { formatOutput } = await import("../../core/format-output.js");
+      const jobs = JSON.parse(await readFile(file, "utf-8"));
+      const criteria = {
+        keywords: (opts.keywords as string[]) ?? [],
+        excludeKeywords: (opts.exclude as string[]) ?? [],
+        location: (opts.location as string) ?? null,
+        workMode: (() => {
+          const modes = [
+            ...(opts.remote ? ["remote" as const] : []),
+            ...(opts.onsite ? ["onsite" as const] : []),
+            ...(opts.hybrid ? ["hybrid" as const] : []),
+          ];
+          return modes.length ? modes : null;
+        })(),
+        departments: (opts.department as string[]) ?? null,
+        role: (opts.role as string[]) ?? null,
+        roleType: (opts.roleType as string[]) ?? null,
+        jobType: (opts.jobType as string[]) ?? null,
+        minExperience: opts.minExperience
+          ? (opts.minExperience as string[]).map((v) => parseInt(v, 10))
+          : null,
+        companyStage: (opts.companyStage as string[]) ?? null,
+        industry: (opts.industry as string[]) ?? null,
+        companySize: (opts.companySize as string[]) ?? null,
+        hasSalary: (opts.hasSalary as boolean) ?? null,
+        hasEquity: (opts.hasEquity as boolean) ?? null,
+        hasInterviewProcess: (opts.hasInterviewProcess as boolean) ?? null,
+        visaSponsorship: (opts.visaSponsorship as boolean) ?? null,
+      };
+      const matched = matchJobs(jobs, criteria);
+      const output = formatOutput(
+        matched,
+        (opts.output as string as "json") ?? "json"
+      );
+      bus.emit("output:text", { text: output });
+    })
+  );
+
+// Handle exit
+bus.on("done", (p) => {
+  process.exitCode = p.exitCode;
+});
+
+program.parse();

package/src/entrypoints/cli/init.ts

@@ -0,0 +1,117 @@
+import { existsSync } from "node:fs";
+import { mkdir, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { bus } from "../../events.js";
+import { companies } from "../../data/companies.js";
+
+const CONFIG_DIR = join(homedir(), ".jobcrawl");
+const CONFIG_FILE = join(CONFIG_DIR, "config.yaml");
+const CREDENTIALS_FILE = join(CONFIG_DIR, "credentials.json");
+const RAW_DIR = join(CONFIG_DIR, "raw");
+
+function generateDefaultConfig(): string {
+  const lines: string[] = [
+    "# jobcrawl config — auto-generated by `jobcrawl init`",
+    "#",
+    "# Aggregators (cross-company search engines):",
+    "#   Run with: jobcrawl crawl-aggregators yc",
+    "#   Or alongside companies: jobcrawl crawl --aggregators yc",
+    "",
+    "aggregators:",
+    "  - type: yc",
+    "    enabled: true",
+    "",
+    "# Companies (add your own or modify existing ones):",
+    "#",
+    "#   Slug-based (recommended):",
+    "#     - company: Company Name",
+    "#       slug: board-slug",
+    "#       provider: greenhouse | ashby | lever  (optional — auto-detected if omitted)",
+    "#       fallback: https://...                 (optional — used when no ATS API matches)",
+    "#",
+    "#   URL-based:",
+    "#     - url: https://example.com/careers",
+    "#       company: Company Name",
+    "",
+    "companies:",
+  ];
+
+  let currentSection = "";
+
+  for (const c of companies) {
+    if (c.section !== currentSection) {
+      currentSection = c.section;
+      if (currentSection) {
+        lines.push("");
+        lines.push(`  # --- ${currentSection} ---`);
+      }
+    }
+
+    lines.push(`  - company: ${c.company}`);
+    lines.push(`    slug: ${c.slug}`);
+    if (c.provider) {
+      lines.push(`    provider: ${c.provider}`);
+    }
+    if (c.fallback) {
+      lines.push(`    fallback: ${c.fallback}`);
+    }
+  }
+
+  lines.push("");
+  lines.push("defaults:");
+  lines.push("  concurrency: 5");
+  lines.push("");
+
+  return lines.join("\n");
+}
+
+export { CONFIG_DIR, CONFIG_FILE, CREDENTIALS_FILE, RAW_DIR };
+
+interface InitOptions {
+  force?: boolean;
+}
+
+export async function initCommand(opts: InitOptions): Promise<void> {
+  const created: string[] = [];
+  const skipped: string[] = [];
+
+  await mkdir(RAW_DIR, { recursive: true, mode: 0o700 });
+  created.push(`${RAW_DIR}/`);
+
+  if (!existsSync(CONFIG_FILE) || opts.force) {
+    await writeFile(CONFIG_FILE, generateDefaultConfig());
+    created.push(CONFIG_FILE);
+  } else {
+    skipped.push(CONFIG_FILE);
+  }
+
+  if (!existsSync(CREDENTIALS_FILE) || opts.force) {
+    const defaultCredentials = {
+      yc: {
+        algoliaAppId: "",
+        algoliaApiKey: "",
+      },
+    };
+    await writeFile(
+      CREDENTIALS_FILE,
+      JSON.stringify(defaultCredentials, null, 2) + "\n",
+      { mode: 0o600 },
+    );
+    created.push(CREDENTIALS_FILE);
+  } else {
+    skipped.push(CREDENTIALS_FILE);
+  }
+
+  const lines: string[] = [];
+  for (const f of created) lines.push(`Created ${f}`);
+  for (const f of skipped) lines.push(`Skipped ${f} (already exists)`);
+  lines.push(
+    "",
+    `Add your YC Algolia credentials to ${CREDENTIALS_FILE}, then run:`,
+    "",
+    "  jobcrawl crawl --keywords \"engineer\"",
+  );
+
+  bus.emit("output:message", { text: lines.join("\n"), style: "success" });
+}