openings 0.1.53 → 0.1.55

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openings",
-  "version": "0.1.53",
+  "version": "0.1.55",
   "description": "A free, candidate-safe job-search substrate for AI agents",
   "license": "MIT",
   "repository": { "type": "git", "url": "git+https://github.com/abhay-avagama/hiring-agent.git" },

package/src/crawler.ts

@@ -27,6 +27,18 @@ interface CrawlerOptions {
    * whose backlog is in the thousands, which 150 a night would never finish. */
   describeLimit?: number;
   describeBudgetMs?: number;
+  /** How old a posting may be and still be worth reading, in days. 0 reads any live role, whatever its date,
+   * including one whose board states no date at all. The nightly crawl keeps the 30-day default so it spends
+   * its small budget on what is new; the catch-up pass lifts it to reach roles still listed but long posted. */
+  describeMaxAgeDays?: number;
+  /**
+   * Whether the snapshot written to disk keeps the description text. A local install needs it: the snapshot is
+   * its only copy. A server crawl reports every source to an aggregator that stores them, so keeping a second
+   * copy only costs memory — on one box the file reached 1.3 GB and parsing it needed more RAM than the machine
+   * had, which killed the crawl for seven nights. The experience read out of each description is always kept,
+   * so a role already read is never fetched twice.
+   */
+  retainDescriptions?: boolean;
   pacingNow?: () => number;
   pacingSleep?: (delayMs: number) => Promise<void>;
   now?: () => Date;
@@ -66,14 +78,18 @@ export function createCrawler(options: CrawlerOptions): Crawler {
   /** Required experience for every job: from its description, the last crawl's reading, or a paced detail fetch for recent roles in describeCountries. */
   async function withExperience(source: Company, jobs: Job[], previous: Job[] | undefined): Promise<Job[]> {
     const known = new Map((previous ?? []).map(normalizeJobExperience).filter((job) => job.experienceVersion === EXPERIENCE_VERSION && job.experience !== undefined).map((job) => [job.id, job]));
-    const since = now().getTime() - 30 * 86_400_000;
+    const maxAgeDays = options.describeMaxAgeDays ?? DESCRIBE_MAX_AGE_DAYS;
+    const since = maxAgeDays > 0 ? now().getTime() - maxAgeDays * 86_400_000 : 0;
+    // An undated posting is still a live posting. Inside a window it cannot be placed, so it waits; with the
+    // window lifted there is no reason to leave it unread.
+    const recentEnough = (job: Job) => (maxAgeDays <= 0 ? true : Date.parse(job.updatedAt ?? "") >= since);
     const out: Job[] = [];
     const pending: number[] = [];
     for (const job of jobs) {
       if (job.description.trim() || job.experienceVersion === EXPERIENCE_VERSION) out.push(normalizeJobExperience(job));
       else if (known.has(job.id)) out.push({ ...job, description: known.get(job.id)!.description, experience: known.get(job.id)!.experience, experienceVersion: EXPERIENCE_VERSION });
       else {
-        if (options.describe && describeRank(job) < describeOrder.length && Date.parse(job.updatedAt ?? "") >= since) pending.push(out.length);
+        if (options.describe && describeRank(job) < describeOrder.length && recentEnough(job)) pending.push(out.length);
         out.push(normalizeJobExperience(job));
       }
     }
@@ -182,7 +198,9 @@ export function createCrawler(options: CrawlerOptions): Crawler {
         cached: considered - eligibleSources.length, deferred: eligibleSources.length - selectedSources.length,
         succeeded, failed: finalFailures, sources: selectedSources.map((source) => metrics.get(source.slug)!),
       };
-      await options.store.write({ version: 1, updatedAt: finishedAt, partitions, lastCrawl: report });
+      // Reporting below still sends the full partitions; only the copy kept on disk is lightened.
+      const stored = options.retainDescriptions === false ? withoutDescriptions(partitions) : partitions;
+      await options.store.write({ version: 1, updatedAt: finishedAt, partitions: stored, lastCrawl: report });
       if (options.onCrawled) {
         const crawled = selectedSources.filter((source) => metrics.get(source.slug)!.status === "succeeded");
         // Four at a time: sending a thousand reports at once made the largest ones time out while the aggregator queued them.
@@ -196,8 +214,21 @@ export function createCrawler(options: CrawlerOptions): Crawler {
 }
 
 // ponytail: per-source caps keep one crawl bounded; a big backlog (first run) drains over a few nights.
+/**
+ * The same partitions with the description text left out. Everything the next crawl needs to avoid re-reading a
+ * description — the experience and its version — is kept, so this costs nothing but the bytes.
+ */
+function withoutDescriptions(partitions: JobSnapshot["partitions"]): JobSnapshot["partitions"] {
+  const out: JobSnapshot["partitions"] = {};
+  for (const [slug, partition] of Object.entries(partitions)) {
+    out[slug] = { ...partition, jobs: partition.jobs.map((job) => (job.description ? { ...job, description: "" } : job)) };
+  }
+  return out;
+}
+
 /** Defaults for the nightly crawl, where the point is to keep every source moving rather than finish any one. */
 const DESCRIBE_LIMIT = 150;
+const DESCRIBE_MAX_AGE_DAYS = 30;
 const DESCRIBE_BUDGET_MS = 180_000;
 
 function partitionTime(value: string | undefined): number {

package/src/local-jobs.ts

@@ -19,6 +19,8 @@ interface LocalJobsOptions {
   describeCountries?: string[];
   describeLimit?: number;
   describeBudgetMs?: number;
+  describeMaxAgeDays?: number;
+  retainDescriptions?: boolean;
   now?: () => Date;
   onCrawled?(source: Company, partition: JobPartition): void | Promise<void>;
 }

package/src/runtime.ts

@@ -40,6 +40,9 @@ export function createRuntime(options: { dataDir?: string; concurrency?: number;
     // The description catch-up pass raises these; unset, the nightly crawl keeps the crawler's own small defaults.
     describeLimit: positiveEnv("OPENINGS_DESCRIBE_LIMIT"),
     describeBudgetMs: positiveEnv("OPENINGS_DESCRIBE_BUDGET_MS"),
+    describeMaxAgeDays: wholeEnv("OPENINGS_DESCRIBE_MAX_AGE_DAYS"),
+    // Server crawls set this off: the aggregator they report to is the system of record for descriptions.
+    retainDescriptions: process.env.OPENINGS_RETAIN_DESCRIPTIONS !== "off",
     onCrawled,
   });
   const recommender = createJobRecommender({ sources: companies, store, crawl: local.crawl });
@@ -107,3 +110,11 @@ function positiveEnv(name: string): number | undefined {
   const value = Number(raw);
   return Number.isInteger(value) && value > 0 ? value : undefined;
 }
+
+/** A whole number from the environment, zero included, or undefined when unset or nonsense. */
+function wholeEnv(name: string): number | undefined {
+  const raw = process.env[name];
+  if (raw === undefined) return undefined;
+  const value = Number(raw);
+  return Number.isInteger(value) && value >= 0 ? value : undefined;
+}

package/src/version.ts

@@ -1 +1 @@
-export const VERSION = "0.1.53";
+export const VERSION = "0.1.55";