@timurtekb/tekjobs 0.24.1

package/scraper/resume.mjs

@@ -0,0 +1,65 @@
+// Resume text extraction: PDF (pdf-parse), DOCX (a minimal ZIP reader + word/document.xml), Markdown and plain text.
+import fs from 'node:fs';
+import path from 'node:path';
+import zlib from 'node:zlib';
+
+export async function extractText(file) {
+  const ext = path.extname(file).toLowerCase();
+  const buf = fs.readFileSync(file);
+  if (ext === '.pdf') return pdfText(buf);
+  if (ext === '.docx') return docxText(buf);
+  if (['.md', '.txt', '.markdown'].includes(ext)) return buf.toString('utf8');
+  throw new Error(`Unsupported resume format "${ext}". Use PDF, DOCX, Markdown or plain text.`);
+}
+
+async function pdfText(buf) {
+  let mod;
+  try { mod = await import('pdf-parse'); } catch { throw new Error('PDF support needs the pdf-parse package: run `npm install` in the TekJobs folder, or import the resume as DOCX, Markdown or text.'); }
+  if (mod.PDFParse) {                                    // pdf-parse v2
+    const parser = new mod.PDFParse({ data: buf });
+    try { const r = await parser.getText(); return normalize(r.text || ''); } finally { await parser.destroy?.(); }
+  }
+  const fn = mod.default || mod;                          // pdf-parse v1
+  const r = await fn(buf);
+  return normalize(r.text || '');
+}
+
+/** DOCX is a ZIP; the body is word/document.xml. Paragraphs become lines, runs are joined, tags stripped. */
+function docxText(buf) {
+  const xml = readZipEntry(buf, 'word/document.xml');
+  if (!xml) throw new Error('Not a DOCX file (no word/document.xml inside).');
+  const text = xml
+    .replace(/<w:tab\/>/g, '\t')
+    .replace(/<\/w:p>/g, '\n')
+    .replace(/<w:br[^>]*\/>/g, '\n')
+    .replace(/<[^>]+>/g, '')
+    .replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&apos;/g, "'");
+  return normalize(text);
+}
+
+function readZipEntry(buf, wanted) {
+  // Walk the central directory (signature 0x02014b50) to find the entry, then read it from its local header.
+  const eocd = buf.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06]));
+  if (eocd < 0) return null;
+  const cdOffset = buf.readUInt32LE(eocd + 16);
+  const count = buf.readUInt16LE(eocd + 10);
+  let p = cdOffset;
+  for (let i = 0; i < count; i++) {
+    if (buf.readUInt32LE(p) !== 0x02014b50) break;
+    const method = buf.readUInt16LE(p + 10);
+    const csize = buf.readUInt32LE(p + 20);
+    const nameLen = buf.readUInt16LE(p + 28), extraLen = buf.readUInt16LE(p + 30), commentLen = buf.readUInt16LE(p + 32);
+    const localOffset = buf.readUInt32LE(p + 42);
+    const name = buf.toString('utf8', p + 46, p + 46 + nameLen);
+    if (name === wanted) {
+      const lnameLen = buf.readUInt16LE(localOffset + 26), lextraLen = buf.readUInt16LE(localOffset + 28);
+      const start = localOffset + 30 + lnameLen + lextraLen;
+      const data = buf.subarray(start, start + csize);
+      return (method === 8 ? zlib.inflateRawSync(data) : data).toString('utf8');
+    }
+    p += 46 + nameLen + extraLen + commentLen;
+  }
+  return null;
+}
+
+const normalize = (s) => s.replace(/\r/g, '').replace(/[ \t]+\n/g, '\n').replace(/\n{3,}/g, '\n\n').trim();

package/scraper/score.mjs

@@ -0,0 +1,114 @@
+const esc = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+/** Pull an annual USD pay range out of free text. Returns { min, max } in dollars or null. */
+export function parseSalary(text = '') {
+  const t = text.replace(/–|—/g, '-');
+  const num = (s, k) => { let n = parseFloat(s.replace(/,/g, '')); if (k) n *= 1000; return n; };
+  const re = /\$\s?(\d{2,3}(?:,\d{3})?(?:\.\d+)?)\s?(k)?\s?(?:-|to|–)\s?\$?\s?(\d{2,3}(?:,\d{3})?(?:\.\d+)?)\s?(k)?/gi;
+  let best = null, m;
+  while ((m = re.exec(t))) {
+    let lo = num(m[1], m[2]), hi = num(m[3], m[4]);
+    if (lo < 1000) lo *= 1000; if (hi < 1000) hi *= 1000;      // "$180-250" style
+    if (lo < 60000 || hi > 1500000 || hi < lo) continue;        // hourly rates, equity %, junk
+    if (!best || hi > best.max) best = { min: lo, max: hi };
+  }
+  return best;
+}
+const wordHit = (hay, term) => new RegExp(`(^|[^a-z0-9])${esc(term)}([^a-z0-9]|$)`, 'i').test(hay);
+
+/**
+ * Returns { score, reasons[], excluded } for one normalized job against the criteria JSON.
+ * `now` is the moment recency is judged from: the scan passes nothing (today); a rescore of an existing note
+ * passes the day the note was found, so the posting keeps the freshness it had when it was scored.
+ */
+export function scoreJob(job, c, { now = Date.now() } = {}) {
+  const title = (job.title || '').toLowerCase();
+  const desc = (job.descriptionText || '').toLowerCase();
+  const loc = (job.location || '').toLowerCase();
+  const reasons = [];
+  let score = 0;
+
+  for (const ex of c.titleExclude || []) {
+    if (title.includes(ex.toLowerCase())) return { score: -999, reasons: [`excluded by title: "${ex}"`], excluded: true };
+  }
+
+  const titleHits = Object.entries(c.titleTerms || {})
+    .filter(([t]) => title.includes(t.toLowerCase()))
+    .sort((a, b) => b[1] - a[1]);
+  if (titleHits.length) {
+    // The best-matching term carries the weight; extra hits add a little and then stop. Uncapped, a title
+    // that happens to contain four of your terms outranks a better job whose title contains one, which is
+    // rewarding vocabulary rather than fit.
+    const extra = Math.min((titleHits.length - 1) * (c.titleExtraPer ?? 5), c.titleExtraCap ?? 10);
+    const pts = titleHits[0][1] + extra;
+    score += pts;
+    reasons.push(`title +${pts}: ${titleHits.map((h) => h[0]).join(', ')}`);
+  } else {
+    const pen = c.noTitleMatchPenalty ?? -40;
+    score += pen;
+    reasons.push(`no title match ${pen}`);
+  }
+
+  for (const [t, w] of Object.entries(c.seniority?.boost || {})) {
+    if (wordHit(title, t)) { score += w; reasons.push(`seniority +${w} (${t})`); break; }
+  }
+  for (const [t, w] of Object.entries(c.seniority?.penalty || {})) {
+    if (wordHit(title, t)) { score += w; reasons.push(`seniority ${w} (${t})`); break; }
+  }
+
+  let d = 0; const dhits = [];
+  for (const [t, w] of Object.entries(c.descTerms || {})) {
+    if (desc.includes(t.toLowerCase())) { d += w; dhits.push(t); }
+  }
+  d = Math.min(d, c.descCap ?? 35);
+  if (dhits.length) { score += d; reasons.push(`description +${d}: ${dhits.join(', ')}`); }
+
+  const L = c.location || {};
+  const isRemote = !!job.remote || /\bremote\b/.test(loc);
+  const bay = (L.bayAreaTerms || []).some((t) => loc.includes(t));
+  const us = (L.usTerms || []).some((t) => wordHit(loc, t));
+  const nonUs = (L.nonUsTerms || []).some((t) => wordHit(loc, t));
+  if (isRemote) { score += L.remoteBoost ?? 0; reasons.push(`remote +${L.remoteBoost ?? 0}`); }
+  else if (L.requireRemote) { score += L.notRemotePenalty ?? -60; reasons.push(`not remote ${L.notRemotePenalty ?? -60}`); }
+  if (bay) { score += L.bayAreaBoost ?? 0; reasons.push(`bay area +${L.bayAreaBoost ?? 0}`); }
+  if (nonUs && !us && !bay) { score += L.nonUsPenalty ?? 0; reasons.push(`non-US location ${L.nonUsPenalty ?? 0}`); }
+
+  const S = c.salary || {};
+  let payBand = 'unknown';
+  if (S.minAnnual && job.salaryMax) {
+    const top = `$${Math.round(job.salaryMax / 1000)}k`;
+    if (job.salaryMax >= S.minAnnual) {
+      payBand = 'floor';
+      // Clearing the floor is worth a fixed amount; clearing it by a lot is worth more. A flat bonus made
+      // a job topping out at the floor and one topping out 40% above it score identically, which is the
+      // opposite of how a person reads the same two numbers. Capped, so pay cannot dominate fit.
+      const over = Math.max(0, job.salaryMax - S.minAnnual);
+      const bonus = Math.min(Math.round((over / 10000) * (S.abovePer10k ?? 1)), S.aboveCap ?? 15);
+      score += (S.meetsBonus ?? 10) + bonus;
+      reasons.push(`pay range tops out at ${top}, at or above floor (+${S.meetsBonus ?? 10}${bonus ? `, +${bonus} for ${Math.round(over / 1000)}k above` : ''})`);
+    }
+    else if (S.stretchAnnual && job.salaryMax >= S.stretchAnnual) { payBand = 'stretch'; score += S.stretchPenalty ?? -8; reasons.push(`pay range tops out at ${top}, stretch band (${S.stretchPenalty ?? -8})`); }
+    else { payBand = 'below'; score += S.belowPenalty ?? -30; reasons.push(`pay range tops out at ${top}, under floor (${S.belowPenalty ?? -30})`); }
+  }
+  job.payBand = payBand;
+
+  if (job.posted) {
+    const days = (now - new Date(job.posted).getTime()) / 86400000;
+    if (Number.isFinite(days)) {
+      // Measured on a real vault: every listing that closed did so within seven days of being found, median
+      // two. A posting is perishable, so the fresh end of this scale needs more resolution than the stale
+      // end — without a days2 tier, something posted today and something posted four weeks ago differ by
+      // five points, which is less than one description keyword.
+      const r = c.recency || {};
+      const add = days <= 2 ? r.days2 ?? r.days7 ?? 0
+        : days <= 7 ? r.days7 ?? 0
+        : days <= 30 ? r.days30 ?? 0
+        : days <= 90 ? r.days90 ?? 0
+        : r.older ?? 0;
+      score += add;
+      reasons.push(`posted ${Math.max(0, Math.round(days))}d ago (${add >= 0 ? '+' : ''}${add})`);
+    }
+  }
+
+  return { score: Math.round(score), reasons, excluded: false, payBand };
+}

package/scraper/sources-email.mjs

@@ -0,0 +1,242 @@
+// Job alert emails, parsed from a folder of saved messages. Same normalized job shape as sources.mjs.
+//
+// This is the front door to the boards that have no usable public API — LinkedIn, Indeed, Otta and the rest
+// all send alert emails, and an email in your own mailbox is your own data. Nothing here contacts those
+// sites: it reads files you put in a folder, and the links it produces are for you to open yourself.
+//
+// Drop `.eml` files into <profile>/Inbox/ (most mail clients save a message with File > Save As, and most
+// can be given a rule that does it automatically). Files are never moved or deleted; the scan's own
+// deduplication by job id stops a message being imported twice.
+//
+// What you get is thin by design. An alert email carries a title, a company, usually a location and rarely
+// a sentence of description, so these rows score on their title almost alone and will sit below the same
+// job fetched from an ATS. That is the right order: this source exists to catch what the others cannot see.
+import fs from 'node:fs';
+import path from 'node:path';
+import { htmlToText, decodeEntities } from './sources.mjs';
+
+// ---------------- a small MIME reader ----------------
+// Enough of RFC 2045 to get the HTML out of a saved alert email, and no more. A full parser is a dependency
+// this project does not want for one job.
+
+function splitHeaders(raw) {
+  const end = raw.search(/\r?\n\r?\n/);
+  const head = end === -1 ? raw : raw.slice(0, end);
+  const body = end === -1 ? '' : raw.slice(end).replace(/^\r?\n\r?\n/, '');
+  const headers = {};
+  // Unfold: a header continues while the next line starts with whitespace.
+  for (const line of head.replace(/\r?\n[ \t]+/g, ' ').split(/\r?\n/)) {
+    const i = line.indexOf(':');
+    if (i > 0) headers[line.slice(0, i).trim().toLowerCase()] = line.slice(i + 1).trim();
+  }
+  return { headers, body };
+}
+
+const decodeQuotedPrintable = (s) =>
+  s.replace(/=\r?\n/g, '').replace(/=([0-9A-Fa-f]{2})/g, (_, h) => String.fromCharCode(parseInt(h, 16)));
+
+function decodeBody(body, encoding = '') {
+  const enc = encoding.toLowerCase();
+  if (enc.includes('base64')) {
+    try { return Buffer.from(body.replace(/\s+/g, ''), 'base64').toString('utf8'); } catch { return body; }
+  }
+  if (enc.includes('quoted-printable')) {
+    // Decode to bytes first, then read as UTF-8: =C3=A9 is two bytes, not two characters.
+    return Buffer.from(decodeQuotedPrintable(body), 'binary').toString('utf8');
+  }
+  return body;
+}
+
+/** The best body part to read: HTML if the message has one, otherwise plain text. */
+function bestPart(raw, depth = 0) {
+  const { headers, body } = splitHeaders(raw);
+  const type = (headers['content-type'] || 'text/plain').toLowerCase();
+
+  if (type.startsWith('multipart/') && depth < 6) {
+    const boundary = (type.match(/boundary="?([^";]+)"?/) || [])[1];
+    if (boundary) {
+      // Non-capturing: a capture group would put its own matches into the split result as undefined entries.
+      const parts = body.split(new RegExp(`--${boundary.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?:--)?\\r?\\n`)).slice(1);
+      const found = parts.filter((p) => p && p.trim()).map((p) => bestPart(p, depth + 1)).filter(Boolean);
+      return found.find((p) => p.type.includes('html')) || found.find((p) => p.type.includes('text')) || null;
+    }
+  }
+  if (!type.startsWith('text/')) return null;
+  return { type, text: decodeBody(body, headers['content-transfer-encoding'] || '') };
+}
+
+/** `=?utf-8?B?...?=` encoded-words, as they appear in Subject and From. */
+function decodeWords(s = '') {
+  return s.replace(/=\?([^?]+)\?([BbQq])\?([^?]*)\?=/g, (_, charset, enc, text) => {
+    try {
+      if (enc.toUpperCase() === 'B') return Buffer.from(text, 'base64').toString('utf8');
+      return Buffer.from(decodeQuotedPrintable(text.replace(/_/g, ' ')), 'binary').toString('utf8');
+    } catch { return text; }
+  });
+}
+
+export function readEmail(raw) {
+  const { headers } = splitHeaders(raw);
+  const part = bestPart(raw);
+  return {
+    from: decodeWords(headers.from || ''),
+    subject: decodeWords(headers.subject || ''),
+    date: headers.date || '',
+    html: part && part.type.includes('html') ? part.text : '',
+    text: part ? part.text : '',
+  };
+}
+
+// ---------------- turning a message into jobs ----------------
+
+/** Tracking parameters make every copy of a link unique, which would defeat deduplication. */
+function cleanUrl(href) {
+  try {
+    const u = new URL(decodeEntities(href.replace(/&amp;/g, '&')));
+    for (const k of [...u.searchParams.keys()]) {
+      // utm_ is a prefix (utm_source, utm_campaign, …); the rest are whole names.
+      if (/^utm_/i.test(k) || /^(trk|tracking|trackingId|refId|midToken|mid|eid|lipi|licu|from|source|ref)$/i.test(k)) u.searchParams.delete(k);
+    }
+    u.hash = '';
+    return u.toString();
+  } catch { return ''; }
+}
+
+/** Anchors, in document order, as { href, text }. */
+function anchors(html) {
+  const out = [];
+  for (const m of html.matchAll(/<a\b[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi)) {
+    // `end` matters: the context for a job is what comes *after* its link. Starting at `index` instead
+    // drags the anchor's own text into the first line whenever the markup has no block break after it,
+    // and the company is then read as part of the title.
+    out.push({ href: m[1], text: decodeEntities(m[2].replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim(), index: m.index ?? 0, end: (m.index ?? 0) + m[0].length });
+  }
+  return out;
+}
+
+/**
+ * Each board's alert email is laid out differently, but all of them repeat one block per job around one
+ * link. `id` pulls a stable identifier out of that link; `clean` rebuilds the canonical URL from it, so the
+ * same job in two different alerts collapses to one row.
+ */
+const BOARDS = [
+  {
+    name: 'linkedin',
+    from: /linkedin\.com/i,
+    link: /linkedin\.com\/(?:comm\/)?jobs\/view\/(\d+)/i,
+    clean: (id) => `https://www.linkedin.com/jobs/view/${id}/`,
+  },
+  {
+    name: 'indeed',
+    from: /indeed\.com/i,
+    link: /indeed\.com\/(?:viewjob|rc\/clk|pagead\/clk)[^"']*?[?&]jk=([0-9a-f]+)/i,
+    clean: (id) => `https://www.indeed.com/viewjob?jk=${id}`,
+  },
+  {
+    // Otta was acquired by Welcome to the Jungle; app.otta.com now redirects to app.welcometothejungle.com.
+    // Both senders and both link shapes are matched, because old alerts keep working and new ones arrive
+    // under the new name.
+    name: 'otta',
+    from: /(otta\.com|welcometothejungle\.com)/i,
+    link: /(?:otta\.com|welcometothejungle\.com)\/jobs\/([A-Za-z0-9_-]+)/i,
+    clean: (id) => `https://app.welcometothejungle.com/jobs/${id}`,
+  },
+  {
+    name: 'wellfound',
+    from: /(wellfound|angel)\.co/i,
+    link: /wellfound\.com\/jobs\/(\d+)/i,
+    clean: (id) => `https://wellfound.com/jobs/${id}`,
+  },
+];
+
+/** Anything else: keep links that look like a job posting on a board we already understand. */
+const GENERIC_LINK = /(greenhouse\.io|lever\.co|ashbyhq\.com|myworkdayjobs\.com|smartrecruiters\.com|workable\.com|breezy\.hr|bamboohr\.com)\/[^"']*/i;
+
+/**
+ * The text that follows a job link, up to the next one — where the company and location live in every
+ * layout examined. Read as plain text so a change of markup does not break it.
+ */
+function contextAfter(html, from, to) {
+  return htmlToText(html.slice(from, to === -1 ? from + 1200 : to))
+    .split('\n')
+    .map((l) => l.trim())
+    .filter(Boolean);
+}
+
+const LOCATION_HINT = /remote|hybrid|on-?site|,\s*[A-Z]{2}\b|United States|United Kingdom|Canada|Germany|India|Ireland|Netherlands|Australia|Singapore/i;
+
+export function jobsFromEmail(raw, file = '') {
+  const mail = readEmail(raw);
+  const html = mail.html || mail.text;
+  if (!html) return [];
+  const board = BOARDS.find((b) => b.from.test(mail.from)) || null;
+  const links = anchors(html);
+  const posted = mail.date && !isNaN(Date.parse(mail.date)) ? new Date(mail.date).toISOString() : null;
+
+  const jobs = [];
+  const seen = new Set();
+  for (let i = 0; i < links.length; i++) {
+    const a = links[i];
+    const hit = board ? a.href.match(board.link) : a.href.match(GENERIC_LINK);
+    if (!hit) continue;
+    const url = board ? board.clean(hit[1]) : cleanUrl(a.href);
+    if (!url || seen.has(url)) continue;
+
+    // The anchor text is the title in every layout examined; anchors that are buttons ("Apply", "View job")
+    // wrap the same link, so the first one with real words wins.
+    const title = a.text.replace(/\s+/g, ' ').trim();
+    if (!title || title.length < 3 || /^(apply|view|see|show)\b/i.test(title)) continue;
+
+    const next = links.slice(i + 1).find((l) => (board ? l.href.match(board.link) : l.href.match(GENERIC_LINK)));
+    const lines = contextAfter(html, a.end, next ? next.index : -1).filter((l) => l && l !== title);
+    const location = lines.find((l) => LOCATION_HINT.test(l) && l.length < 80) || '';
+    const company = lines.find((l) => l !== location && l.length < 60 && !/^\W/.test(l)) || '';
+
+    seen.add(url);
+    jobs.push({
+      id: `mail:${board ? board.name : 'link'}:${url}`,
+      source: `email:${board ? board.name : 'link'}`,
+      company: company || 'Unknown (from an alert email)',
+      title,
+      url,
+      location,
+      remote: /\bremote\b/i.test(`${location} ${title}`),
+      posted,
+      // Alert emails carry a snippet at best. The subject is kept so the note says where the row came from.
+      descriptionHtml: `<p>Imported from an alert email${board ? ` (${board.name})` : ''}${file ? `: <code>${path.basename(file)}</code>` : ''}.</p><p>${decodeEntities(mail.subject)}</p>`,
+      salary: '',
+      department: '',
+      employmentType: '',
+    });
+  }
+  return jobs;
+}
+
+/**
+ * Every `.eml` in the inbox folder, as jobs. Never writes, moves or deletes a message: the folder is the
+ * person's, and the scan already refuses to rewrite a note it has seen before.
+ */
+export async function fetchEmailInbox(criteria = {}) {
+  const cfg = criteria.email || {};
+  const dir = cfg.dir || (criteria._inbox ?? '');
+  if (!dir) return { ok: false, jobs: [], error: 'no inbox folder configured' };
+  if (!fs.existsSync(dir)) return { ok: false, jobs: [], error: `no folder at ${dir} — create it and save alert emails into it as .eml` };
+
+  const files = fs.readdirSync(dir).filter((f) => /\.(eml|txt|mht|mhtml)$/i.test(f));
+  if (!files.length) return { ok: false, jobs: [], error: `no .eml files in ${dir}` };
+
+  const all = [];
+  const failures = [];
+  for (const f of files.slice(0, cfg.maxFiles ?? 200)) {
+    const full = path.join(dir, f);
+    try {
+      all.push(...jobsFromEmail(fs.readFileSync(full, 'utf8'), full));
+    } catch (e) {
+      failures.push(`${f}: ${e.message}`);
+    }
+  }
+  const byUrl = new Map();
+  for (const j of all) if (!byUrl.has(j.url)) byUrl.set(j.url, j);
+  if (!byUrl.size) return { ok: false, jobs: [], error: failures.length ? `no jobs found; ${failures[0]}` : `no job links found in ${files.length} message(s)` };
+  return { ok: true, jobs: [...byUrl.values()] };
+}