jobcrawl 0.1.0

package/src/entrypoints/cli/app.tsx

@@ -0,0 +1,173 @@
+import React, { useState, useEffect } from "react";
+import { Text, Box, useApp } from "ink";
+import { bus } from "../../events.js";
+import { CrawlView } from "./components/crawl-view.js";
+import { DiscoverView } from "./components/discover-view.js";
+
+type ViewState =
+  | { type: "idle" }
+  | { type: "crawl" }
+  | { type: "discover" }
+  | { type: "json"; data: unknown }
+  | { type: "text"; text: string }
+  | { type: "table"; headers: string[]; rows: string[][]; footer?: string }
+  | { type: "message"; text: string; style?: "success" | "warning" | "dim" }
+  | { type: "error"; message: string };
+
+export function App({ run }: { run: () => Promise<void> }) {
+  const { exit } = useApp();
+  const [view, setView] = useState<ViewState>({ type: "idle" });
+  const [output, setOutput] = useState<ViewState | null>(null);
+  const [error, setError] = useState<string | null>(null);
+
+  useEffect(() => {
+    // Crawl events → switch to crawl view
+    const onCrawlStart = () => setView({ type: "crawl" });
+
+    // Discover events → switch to discover view
+    const onDiscoverSearching = () => setView({ type: "discover" });
+
+    // Output events → store separately so they don't replace the progress view
+    const onOutputJson = (p: { data: unknown }) =>
+      setOutput({ type: "json", data: p.data });
+    const onOutputText = (p: { text: string }) =>
+      setOutput({ type: "text", text: p.text });
+    const onOutputTable = (p: {
+      headers: string[];
+      rows: string[][];
+      footer?: string;
+    }) => setOutput({ type: "table", ...p });
+    const onOutputMessage = (p: {
+      text: string;
+      style?: "success" | "warning" | "dim";
+    }) => setOutput({ type: "message", ...p });
+
+    // Error + lifecycle
+    const onError = (p: { message: string }) => setError(p.message);
+    const onDone = () => {
+      setTimeout(() => exit(), 16);
+    };
+
+    bus.on("crawl:start", onCrawlStart);
+    bus.on("discover:searching", onDiscoverSearching);
+    bus.on("output:json", onOutputJson);
+    bus.on("output:text", onOutputText);
+    bus.on("output:table", onOutputTable);
+    bus.on("output:message", onOutputMessage);
+    bus.on("error", onError);
+    bus.on("done", onDone);
+
+    // Execute command after all listeners are attached
+    run()
+      .then(() => bus.emit("done", { exitCode: 0 }))
+      .catch((err: unknown) => {
+        bus.emit("error", {
+          message: err instanceof Error ? err.message : String(err),
+        });
+        bus.emit("done", { exitCode: 1 });
+      });
+
+    return () => {
+      bus.off("crawl:start", onCrawlStart);
+      bus.off("discover:searching", onDiscoverSearching);
+      bus.off("output:json", onOutputJson);
+      bus.off("output:text", onOutputText);
+      bus.off("output:table", onOutputTable);
+      bus.off("output:message", onOutputMessage);
+      bus.off("error", onError);
+      bus.off("done", onDone);
+    };
+  }, [exit]);
+
+  return (
+    <Box flexDirection="column">
+      {error && <Text color="red">Error: {error}</Text>}
+      {/* Always mount CrawlView so it subscribes before run() fires events.
+          Child useEffect runs before parent useEffect, guaranteeing the
+          subscription is active when the first url:fetching arrives. */}
+      <CrawlView />
+      <ViewRenderer view={view} />
+      {output && <ViewRenderer view={output} />}
+    </Box>
+  );
+}
+
+function ViewRenderer({ view }: { view: ViewState }) {
+  switch (view.type) {
+    case "idle":
+      return null;
+
+    case "crawl":
+      return null;
+
+    case "discover":
+      return <DiscoverView />;
+
+    case "json":
+      return <Text>{JSON.stringify(view.data, null, 2)}</Text>;
+
+    case "text":
+      return <Text>{view.text}</Text>;
+
+    case "table":
+      return (
+        <TableView
+          headers={view.headers}
+          rows={view.rows}
+          footer={view.footer}
+        />
+      );
+
+    case "message": {
+      const color =
+        view.style === "success"
+          ? "green"
+          : view.style === "warning"
+            ? "yellow"
+            : undefined;
+      const dim = view.style === "dim";
+      return (
+        <Text color={color} dimColor={dim}>
+          {view.text}
+        </Text>
+      );
+    }
+
+    case "error":
+      return <Text color="red">{view.message}</Text>;
+  }
+}
+
+function TableView({
+  headers,
+  rows,
+  footer,
+}: {
+  headers: string[];
+  rows: string[][];
+  footer?: string;
+}) {
+  const widths = headers.map((h, i) =>
+    Math.min(50, Math.max(h.length, ...rows.map((r) => (r[i] ?? "").length)))
+  );
+
+  const headerLine = headers.map((h, i) => h.padEnd(widths[i])).join(" | ");
+  const separator = widths.map((w) => "-".repeat(w)).join("-+-");
+
+  return (
+    <Box flexDirection="column">
+      <Text bold>{headerLine}</Text>
+      <Text>{separator}</Text>
+      {rows.map((row, i) => (
+        <Text key={i}>
+          {row
+            .map((cell, j) =>
+              (cell ?? "").slice(0, widths[j]).padEnd(widths[j])
+            )
+            .join(" | ")}
+        </Text>
+      ))}
+      {footer && <Text dimColor>{footer}</Text>}
+    </Box>
+  );
+}

package/src/entrypoints/cli/components/crawl-view.tsx

@@ -0,0 +1,163 @@
+import React, { useState, useEffect } from "react";
+import { Text, Box } from "ink";
+import Spinner from "ink-spinner";
+import { bus } from "../../../events.js";
+
+type StepLine = {
+  text: string;
+  status: "active" | "done" | "failed";
+};
+
+type UrlThread = {
+  company: string;
+  steps: StepLine[];
+};
+
+interface CrawlSummary {
+  totalJobs: number;
+  totalMatched: number;
+  totalUrls: number;
+  failedUrls: number;
+}
+
+export function CrawlView() {
+  const [threads, setThreads] = useState<Map<string, UrlThread>>(new Map());
+  const [summary, setSummary] = useState<CrawlSummary | null>(null);
+
+  useEffect(() => {
+    /** Mark the current active step as done, then push a new active step. */
+    const pushStep = (urlId: string, text: string) => {
+      setThreads((prev) => {
+        const next = new Map(prev);
+        const thread = next.get(urlId);
+        if (!thread) return prev;
+        const steps = thread.steps.map((s) =>
+          s.status === "active" ? { ...s, status: "done" as const } : s
+        );
+        steps.push({ text, status: "active" });
+        next.set(urlId, { ...thread, steps });
+        return next;
+      });
+    };
+
+    /** Mark all active steps as done/failed. */
+    const finalize = (urlId: string, status: "done" | "failed", finalText?: string) => {
+      setThreads((prev) => {
+        const next = new Map(prev);
+        const thread = next.get(urlId);
+        if (!thread) return prev;
+        const steps = thread.steps.map((s) =>
+          s.status === "active" ? { ...s, status } : s
+        );
+        if (finalText) steps.push({ text: finalText, status });
+        next.set(urlId, { ...thread, steps });
+        return next;
+      });
+    };
+
+    const onCrawlStart = () => {};
+
+    const onFetching = (p: { urlId: string; url: string; company: string }) => {
+      setThreads((prev) => {
+        const next = new Map(prev);
+        next.set(p.urlId, {
+          company: p.company,
+          steps: [{ text: "fetching", status: "active" }],
+        });
+        return next;
+      });
+    };
+
+    const onDetecting = (p: { urlId: string; provider: string }) => {
+      pushStep(p.urlId, `detected ${p.provider}`);
+    };
+
+    const onExtracting = (p: { urlId: string; jobCount?: number }) => {
+      const suffix = p.jobCount !== undefined ? ` (${p.jobCount} jobs)` : "";
+      pushStep(p.urlId, `extracting${suffix}`);
+    };
+
+    const onMatching = (p: { urlId: string; matched: number; total: number }) => {
+      pushStep(p.urlId, `matching ${p.matched}/${p.total}`);
+    };
+
+    const onDone = (p: { urlId: string; company: string; matched: number; total: number }) => {
+      finalize(p.urlId, "done", `${p.matched}/${p.total} matched`);
+    };
+
+    const onFailed = (p: { urlId: string; company: string; error: string }) => {
+      finalize(p.urlId, "failed", p.error);
+    };
+
+    const onComplete = (p: CrawlSummary) => setSummary(p);
+
+    bus.on("crawl:start", onCrawlStart);
+    bus.on("url:fetching", onFetching);
+    bus.on("url:detecting", onDetecting);
+    bus.on("url:extracting", onExtracting);
+    bus.on("url:matching", onMatching);
+    bus.on("url:done", onDone);
+    bus.on("url:failed", onFailed);
+    bus.on("crawl:complete", onComplete);
+
+    return () => {
+      bus.off("crawl:start", onCrawlStart);
+      bus.off("url:fetching", onFetching);
+      bus.off("url:detecting", onDetecting);
+      bus.off("url:extracting", onExtracting);
+      bus.off("url:matching", onMatching);
+      bus.off("url:done", onDone);
+      bus.off("url:failed", onFailed);
+      bus.off("crawl:complete", onComplete);
+    };
+  }, []);
+
+  return (
+    <Box flexDirection="column">
+      {[...threads.entries()].map(([urlId, thread]) => (
+        <Box key={urlId} flexDirection="column" marginBottom={1}>
+          <Text bold>{thread.company}</Text>
+          {thread.steps.map((step, i) => (
+            <StepRow key={i} step={step} />
+          ))}
+        </Box>
+      ))}
+
+      {summary && (
+        <Box marginTop={0}>
+          <Text color={summary.totalMatched > 0 ? "green" : "yellow"}>
+            {"\u2714"} {summary.totalMatched} jobs matched across{" "}
+            {summary.totalUrls} URL{summary.totalUrls === 1 ? "" : "s"}
+            {summary.failedUrls > 0 && (
+              <Text color="yellow"> ({summary.failedUrls} failed)</Text>
+            )}
+          </Text>
+        </Box>
+      )}
+    </Box>
+  );
+}
+
+function StepRow({ step }: { step: StepLine }) {
+  if (step.status === "done") {
+    return (
+      <Text>
+        {"  "}<Text color="green">{"\u2714"}</Text> <Text dimColor>{step.text}</Text>
+      </Text>
+    );
+  }
+
+  if (step.status === "failed") {
+    return (
+      <Text>
+        {"  "}<Text color="red">{"\u2718"}</Text> <Text color="red">{step.text}</Text>
+      </Text>
+    );
+  }
+
+  return (
+    <Text>
+      {"  "}<Text color="cyan"><Spinner type="dots" /></Text> {step.text}
+    </Text>
+  );
+}

package/src/entrypoints/cli/components/discover-view.tsx

@@ -0,0 +1,138 @@
+import React, { useState, useEffect } from "react";
+import { Text, Box } from "ink";
+import Spinner from "ink-spinner";
+import { bus } from "../../../events.js";
+
+type CompanyStatus =
+  | { state: "searching" }
+  | { state: "verifying"; url: string }
+  | { state: "found"; url: string }
+  | { state: "not-found"; reason: string };
+
+export function DiscoverView() {
+  const [companies, setCompanies] = useState<Map<string, CompanyStatus>>(
+    new Map()
+  );
+  const [summary, setSummary] = useState<{
+    found: number;
+    notFound: number;
+    total: number;
+  } | null>(null);
+
+  useEffect(() => {
+    const onSearching = (p: { company: string }) => {
+      setCompanies((prev) => {
+        const next = new Map(prev);
+        next.set(p.company, { state: "searching" });
+        return next;
+      });
+    };
+
+    const onVerifying = (p: { company: string; url: string }) => {
+      setCompanies((prev) => {
+        const next = new Map(prev);
+        next.set(p.company, { state: "verifying", url: p.url });
+        return next;
+      });
+    };
+
+    const onFound = (p: { company: string; url: string }) => {
+      setCompanies((prev) => {
+        const next = new Map(prev);
+        next.set(p.company, { state: "found", url: p.url });
+        return next;
+      });
+    };
+
+    const onNotFound = (p: { company: string; reason: string }) => {
+      setCompanies((prev) => {
+        const next = new Map(prev);
+        next.set(p.company, { state: "not-found", reason: p.reason });
+        return next;
+      });
+    };
+
+    const onComplete = (p: {
+      found: number;
+      notFound: number;
+      total: number;
+    }) => {
+      setSummary(p);
+    };
+
+    bus.on("discover:searching", onSearching);
+    bus.on("discover:verifying", onVerifying);
+    bus.on("discover:found", onFound);
+    bus.on("discover:not-found", onNotFound);
+    bus.on("discover:complete", onComplete);
+
+    return () => {
+      bus.off("discover:searching", onSearching);
+      bus.off("discover:verifying", onVerifying);
+      bus.off("discover:found", onFound);
+      bus.off("discover:not-found", onNotFound);
+      bus.off("discover:complete", onComplete);
+    };
+  }, []);
+
+  return (
+    <Box flexDirection="column">
+      {[...companies.entries()].map(([company, status]) => (
+        <CompanyRow key={company} company={company} status={status} />
+      ))}
+
+      {summary && (
+        <Box marginTop={1}>
+          <Text color="green">
+            {"\u2713"} {summary.found}/{summary.total} career pages found
+          </Text>
+        </Box>
+      )}
+    </Box>
+  );
+}
+
+function CompanyRow({
+  company,
+  status,
+}: {
+  company: string;
+  status: CompanyStatus;
+}) {
+  switch (status.state) {
+    case "searching":
+      return (
+        <Text>
+          <Spinner type="dots" /> {company} <Text dimColor>searching...</Text>
+        </Text>
+      );
+
+    case "verifying":
+      return (
+        <Text>
+          <Spinner type="dots" /> {company}{" "}
+          <Text dimColor>verifying {status.url}</Text>
+        </Text>
+      );
+
+    case "found":
+      return (
+        <Text>
+          <Text color="green">{"\u2713"}</Text> {company}{" "}
+          <Text dimColor>
+            {"\u2192"} {status.url}
+          </Text>
+        </Text>
+      );
+
+    case "not-found":
+      return (
+        <Text>
+          <Text color="red">{"\u2717"}</Text> {company}{" "}
+          <Text color="red">
+            {"\u2014"} {status.reason}
+          </Text>
+        </Text>
+      );
+  }
+}

package/src/entrypoints/cli/crawl-aggregators.ts

@@ -0,0 +1,112 @@
+import { bus } from "../../events.js";
+import { extractViaAggregator } from "../../core/extract-jobs.js";
+import { matchJobs } from "../../core/match-jobs.js";
+import { formatOutput } from "../../core/format-output.js";
+import type {
+  Aggregator,
+  SearchCriteria,
+  OutputFormat,
+} from "../../types/index.js";
+
+interface CrawlAggregatorsOptions {
+  keywords?: string[];
+  exclude?: string[];
+  location?: string;
+  remote?: boolean;
+  onsite?: boolean;
+  hybrid?: boolean;
+  department?: string[];
+  role?: string[];
+  roleType?: string[];
+  jobType?: string[];
+  minExperience?: string[];
+  companyStage?: string[];
+  industry?: string[];
+  companySize?: string[];
+  hasSalary?: boolean;
+  hasEquity?: boolean;
+  hasInterviewProcess?: boolean;
+  visaSponsorship?: boolean;
+  output?: OutputFormat;
+  out?: string;
+  saveRaw?: boolean;
+}
+
+export async function crawlAggregatorsCommand(
+  aggregatorNames: string[],
+  opts: CrawlAggregatorsOptions
+): Promise<void> {
+  const valid: Aggregator[] = [];
+  for (const name of aggregatorNames) {
+    if (name === "yc") {
+      valid.push(name);
+    } else {
+      throw new Error(`Unknown aggregator: "${name}". Available: yc`);
+    }
+  }
+
+  if (valid.length === 0) {
+    throw new Error("No aggregators specified. Available: yc");
+  }
+
+  const criteria: SearchCriteria = {
+    keywords: opts.keywords ?? [],
+    excludeKeywords: opts.exclude ?? [],
+    location: opts.location ?? null,
+    workMode: buildWorkMode(opts),
+    departments: opts.department ?? null,
+    role: opts.role ?? null,
+    roleType: opts.roleType ?? null,
+    jobType: opts.jobType ?? null,
+    minExperience: opts.minExperience
+      ? opts.minExperience.map((v) => parseInt(v, 10))
+      : null,
+    companyStage: opts.companyStage ?? null,
+    industry: opts.industry ?? null,
+    companySize: opts.companySize ?? null,
+    hasSalary: opts.hasSalary ?? null,
+    hasEquity: opts.hasEquity ?? null,
+    hasInterviewProcess: opts.hasInterviewProcess ?? null,
+    visaSponsorship: opts.visaSponsorship ?? null,
+  };
+
+  const format = opts.output ?? "json";
+  const allJobs = [];
+
+  for (const aggregator of valid) {
+    bus.emit("aggregator:start", { type: aggregator });
+    try {
+      const jobs = await extractViaAggregator(aggregator, criteria, opts.saveRaw);
+      const matched = matchJobs(jobs, criteria);
+      bus.emit("aggregator:done", { type: aggregator, jobCount: matched.length });
+      allJobs.push(...matched);
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      bus.emit("aggregator:failed", { type: aggregator, error: message });
+      throw new Error(`Aggregator "${aggregator}" failed: ${message}`);
+    }
+  }
+
+  const output = formatOutput(allJobs, format);
+
+  if (opts.out) {
+    const { writeFile } = await import("node:fs/promises");
+    await writeFile(opts.out, output + "\n");
+    bus.emit("output:message", {
+      text: `Wrote ${allJobs.length} jobs to ${opts.out}`,
+      style: "success",
+    });
+  } else {
+    bus.emit("output:text", { text: output });
+  }
+}
+
+function buildWorkMode(
+  opts: CrawlAggregatorsOptions
+): ("remote" | "onsite" | "hybrid")[] | null {
+  const modes: ("remote" | "onsite" | "hybrid")[] = [];
+  if (opts.remote) modes.push("remote");
+  if (opts.onsite) modes.push("onsite");
+  if (opts.hybrid) modes.push("hybrid");
+  return modes.length > 0 ? modes : null;
+}

package/src/entrypoints/cli/crawl-url.ts

@@ -0,0 +1,87 @@
+import { bus } from "../../events.js";
+import { processTarget } from "../../threads/process-url.js";
+import { formatOutput } from "../../core/format-output.js";
+import type { SearchCriteria, OutputFormat } from "../../types/index.js";
+
+interface CrawlUrlOptions {
+  keywords?: string[];
+  exclude?: string[];
+  location?: string;
+  remote?: boolean;
+  onsite?: boolean;
+  hybrid?: boolean;
+  department?: string[];
+  role?: string[];
+  roleType?: string[];
+  jobType?: string[];
+  minExperience?: string[];
+  companyStage?: string[];
+  industry?: string[];
+  companySize?: string[];
+  hasSalary?: boolean;
+  hasEquity?: boolean;
+  hasInterviewProcess?: boolean;
+  visaSponsorship?: boolean;
+  output?: OutputFormat;
+  out?: string;
+  saveRaw?: boolean;
+}
+
+export async function crawlUrlCommand(
+  url: string,
+  opts: CrawlUrlOptions
+): Promise<void> {
+  const criteria: SearchCriteria = {
+    keywords: opts.keywords ?? [],
+    excludeKeywords: opts.exclude ?? [],
+    location: opts.location ?? null,
+    workMode: buildWorkMode(opts),
+    departments: opts.department ?? null,
+    role: opts.role ?? null,
+    roleType: opts.roleType ?? null,
+    jobType: opts.jobType ?? null,
+    minExperience: opts.minExperience
+      ? opts.minExperience.map((v) => parseInt(v, 10))
+      : null,
+    companyStage: opts.companyStage ?? null,
+    industry: opts.industry ?? null,
+    companySize: opts.companySize ?? null,
+    hasSalary: opts.hasSalary ?? null,
+    hasEquity: opts.hasEquity ?? null,
+    hasInterviewProcess: opts.hasInterviewProcess ?? null,
+    visaSponsorship: opts.visaSponsorship ?? null,
+  };
+
+  const format = opts.output ?? "json";
+
+  const result = await processTarget({ url }, criteria, {
+    saveRaw: opts.saveRaw,
+  });
+
+  if (result.error) {
+    throw new Error(`Failed to crawl ${url}: ${result.error}`);
+  }
+
+  const output = formatOutput(result.jobs, format);
+
+  if (opts.out) {
+    const { writeFile } = await import("node:fs/promises");
+    await writeFile(opts.out, output + "\n");
+    bus.emit("output:message", {
+      text: `Wrote ${result.jobs.length} jobs to ${opts.out}`,
+      style: "success",
+    });
+  } else {
+    bus.emit("output:text", { text: output });
+  }
+}
+
+function buildWorkMode(
+  opts: CrawlUrlOptions
+): ("remote" | "onsite" | "hybrid")[] | null {
+  const modes: ("remote" | "onsite" | "hybrid")[] = [];
+  if (opts.remote) modes.push("remote");
+  if (opts.onsite) modes.push("onsite");
+  if (opts.hybrid) modes.push("hybrid");
+  return modes.length > 0 ? modes : null;
+}