openings 0.1.50 → 0.1.52

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openings",
-  "version": "0.1.50",
+  "version": "0.1.52",
   "description": "A free, candidate-safe job-search substrate for AI agents",
   "license": "MIT",
   "repository": { "type": "git", "url": "git+https://github.com/abhay-avagama/hiring-agent.git" },

package/src/crawler.ts

@@ -22,6 +22,11 @@ interface CrawlerOptions {
   describe?(source: Company, job: Job): Promise<string>;
   /** Only roles open to these countries, posted in the last 30 days, get a description fetch. Empty: none do. */
   describeCountries?: string[];
+  /** How many descriptions one source may be asked for in a run, and how long that may take. The nightly crawl
+   * keeps both small so no employer holds up the pass; the description catch-up raises them to reach an employer
+   * whose backlog is in the thousands, which 150 a night would never finish. */
+  describeLimit?: number;
+  describeBudgetMs?: number;
   pacingNow?: () => number;
   pacingSleep?: (delayMs: number) => Promise<void>;
   now?: () => Date;
@@ -45,26 +50,47 @@ export function createCrawler(options: CrawlerOptions): Crawler {
   const pacingSleep = options.pacingSleep ?? ((delayMs: number) => new Promise<void>((resolve) => setTimeout(resolve, delayMs)));
   let previousStart: number | undefined;
   let pacingGate = Promise.resolve();
-  const describeCountries = new Set(options.describeCountries ?? []);
+  /** Ordered, because the budget below is spent in this order: the first country named is served first. */
+  const describeOrder = options.describeCountries ?? [];
+  const describeCountries = new Set(describeOrder);
+  /** Where a job sits in the country priority; lower is sooner. Jobs outside the list are never fetched. */
+  function describeRank(job: Job): number {
+    let best = describeOrder.length;
+    for (const code of job.eligibleCountries) {
+      const rank = describeOrder.indexOf(code);
+      if (rank >= 0 && rank < best) best = rank;
+    }
+    return best;
+  }
 
   /** Required experience for every job: from its description, the last crawl's reading, or a paced detail fetch for recent roles in describeCountries. */
   async function withExperience(source: Company, jobs: Job[], previous: Job[] | undefined): Promise<Job[]> {
     const known = new Map((previous ?? []).map(normalizeJobExperience).filter((job) => job.experienceVersion === EXPERIENCE_VERSION && job.experience !== undefined).map((job) => [job.id, job]));
     const since = now().getTime() - 30 * 86_400_000;
-    const deadline = Date.now() + DESCRIBE_BUDGET_MS;
-    let budget = DESCRIBE_LIMIT;
     const out: Job[] = [];
+    const pending: number[] = [];
     for (const job of jobs) {
       if (job.description.trim() || job.experienceVersion === EXPERIENCE_VERSION) out.push(normalizeJobExperience(job));
       else if (known.has(job.id)) out.push({ ...job, description: known.get(job.id)!.description, experience: known.get(job.id)!.experience, experienceVersion: EXPERIENCE_VERSION });
-      else if (options.describe && budget > 0 && Date.now() < deadline && job.eligibleCountries.some((code) => describeCountries.has(code)) && Date.parse(job.updatedAt ?? "") >= since) {
-        budget -= 1;
-        try {
-          if (options.workdayPageDelayMs) await pacingSleep(options.workdayPageDelayMs);
-          const description = await options.describe(source, job);
-          out.push(normalizeJobExperience({ ...job, description }));
-        } catch { out.push(normalizeJobExperience(job)); } // unread; the next crawl tries again
-      } else out.push(normalizeJobExperience(job));
+      else {
+        if (options.describe && describeRank(job) < describeOrder.length && Date.parse(job.updatedAt ?? "") >= since) pending.push(out.length);
+        out.push(normalizeJobExperience(job));
+      }
+    }
+    // The budget is small and shared, so it is spent by country priority rather than in whatever order the board
+    // listed its jobs. Without this, an employer with 4 India roles among 11,581 American ones never reads the four.
+    // Ties keep the board's own order, which is newest first on every provider that dates its postings.
+    pending.sort((left, right) => describeRank(jobs[left]!) - describeRank(jobs[right]!) || left - right);
+    const deadline = Date.now() + (options.describeBudgetMs ?? DESCRIBE_BUDGET_MS);
+    let budget = options.describeLimit ?? DESCRIBE_LIMIT;
+    for (const index of pending) {
+      if (budget <= 0 || Date.now() >= deadline) break;
+      budget -= 1;
+      try {
+        if (options.workdayPageDelayMs) await pacingSleep(options.workdayPageDelayMs);
+        const description = await options.describe!(source, jobs[index]!);
+        out[index] = normalizeJobExperience({ ...jobs[index]!, description });
+      } catch { /* unread; the next crawl tries again */ }
     }
     return out;
   }
@@ -170,6 +196,7 @@ export function createCrawler(options: CrawlerOptions): Crawler {
 }
 
 // ponytail: per-source caps keep one crawl bounded; a big backlog (first run) drains over a few nights.
+/** Defaults for the nightly crawl, where the point is to keep every source moving rather than finish any one. */
 const DESCRIBE_LIMIT = 150;
 const DESCRIBE_BUDGET_MS = 180_000;
 

package/src/local-jobs.ts

@@ -17,6 +17,8 @@ interface LocalJobsOptions {
   workdayCountries?: string[];
   describe?(source: Company, job: Job): Promise<string>;
   describeCountries?: string[];
+  describeLimit?: number;
+  describeBudgetMs?: number;
   now?: () => Date;
   onCrawled?(source: Company, partition: JobPartition): void | Promise<void>;
 }

package/src/runtime.ts

@@ -37,6 +37,9 @@ export function createRuntime(options: { dataDir?: string; concurrency?: number;
     // Server crawls set OPENINGS_DESCRIBE_COUNTRIES (IN) to read required experience from Workday detail pages; installs skip the extra requests.
     describe: (source, job) => fetchJobDescription(source, job, globalThis.fetch),
     describeCountries: (process.env.OPENINGS_DESCRIBE_COUNTRIES ?? "").split(",").map((code) => code.trim().toUpperCase()).filter((code) => /^[A-Z]{2}$/.test(code)),
+    // The description catch-up pass raises these; unset, the nightly crawl keeps the crawler's own small defaults.
+    describeLimit: positiveEnv("OPENINGS_DESCRIBE_LIMIT"),
+    describeBudgetMs: positiveEnv("OPENINGS_DESCRIBE_BUDGET_MS"),
     onCrawled,
   });
   const recommender = createJobRecommender({ sources: companies, store, crawl: local.crawl });
@@ -96,3 +99,11 @@ export function createHostedRuntime(options: { sources: Company[]; store: Snapsh
     prepareJobSearch: preparation.prepare, getJobCoverage: coverage.getCoverage, recommend: recommender.recommend, analyzeJobFit: analyzer.analyze, optimizeResume: optimizer.optimize,
   };
 }
+
+/** A positive whole number from the environment, or undefined when unset or nonsense. */
+function positiveEnv(name: string): number | undefined {
+  const raw = process.env[name];
+  if (raw === undefined) return undefined;
+  const value = Number(raw);
+  return Number.isInteger(value) && value > 0 ? value : undefined;
+}

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = "0.1.50";
+export const VERSION = "0.1.52";