linkedin-secret-sauce 0.12.1 → 0.12.3

package/docs/INTEGRATION.md

@@ -0,0 +1,405 @@
+# LinkedIn Secret Sauce — Integration Guide
+
+Use this doc as a drop-in "How to use" for other apps that want to call LinkedIn Sales Navigator and enrich emails via this library.
+
+- Runtime: Node.js 18+ (uses global `fetch`)
+- Package: `linkedin-secret-sauce`
+- Scope: server-side use only
+
+## Install
+
+- pnpm: `pnpm add linkedin-secret-sauce@^0.12.0`
+- npm: `npm i linkedin-secret-sauce@^0.12.0`
+- GitHub (fallback/private): `pnpm add github:enerage/LinkedInSecretSauce#v0.12.2`
+
+Versioning
+- Recommend a semver range in consumers, e.g. `^0.12.0`.
+- In monorepos, add a root override to force-sync if needed:
+  - package.json (root): `{ "overrides": { "linkedin-secret-sauce": "0.12.2" } }`
+  - Update across workspace: `pnpm -r up linkedin-secret-sauce`
+
+## Environment
+
+Add to your app's `.env` or secrets:
+
+```
+COSIALL_API_URL=https://api.cosiall.com            # Your Cosiall base URL
+COSIALL_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx   # Your Cosiall API key
+LINKEDIN_PROXY_STRING=proxy.host:8080:user:pass    # Optional; supports URL forms too
+LOG_LEVEL=info                                     # debug|info|warn|error
+```
+
+## Initialize Once
+
+```ts
+// bootstrap/linkedin.ts
+import { initializeLinkedInClient, type LinkedInClientConfig } from 'linkedin-secret-sauce';
+
+export async function initLinkedIn() {
+  const cfg: LinkedInClientConfig = {
+    cosiallApiUrl: process.env.COSIALL_API_URL!,
+    cosiallApiKey: process.env.COSIALL_API_KEY!,
+    proxyString: process.env.LINKEDIN_PROXY_STRING, // optional
+    logLevel: (process.env.LOG_LEVEL as any) || 'info',
+
+    // Optional overrides (defaults shown)
+    profileCacheTtl: 15 * 60 * 1000,
+    searchCacheTtl: 15 * 60 * 1000,
+    companyCacheTtl: 10 * 60 * 1000,
+    typeaheadCacheTtl: 60 * 60 * 1000,
+    cookieRefreshInterval: 15 * 60 * 1000,
+    cookieFreshnessWindow: 24 * 60 * 60 * 1000,
+    maxRetries: 2,
+    retryDelayMs: 700,
+    accountCooldownMs: 5000,
+    maxFailuresBeforeCooldown: 3,
+    maxRequestHistory: 500,
+
+    // NEW: Advanced initialization options
+    eagerInitialization: true,  // Default: true - Fetch cookies during init (recommended)
+    fallbackWithoutProxyOnError: false,  // Default: false - Retry without proxy on 502/503/504
+    
+    // Optional error handler for initialization failures
+    onInitializationError: (error) => {
+      console.error('LinkedIn init failed:', error);
+      // Handle gracefully instead of throwing
+    },
+    
+    // Optional Sentry integration for error tracking
+    sentryClient: mySentryClient,  // Pass your Sentry client instance
+  };
+
+  await initializeLinkedInClient(cfg);
+}
+```
+
+Call `await initLinkedIn()` once at process startup (before making any API calls).
+
+### Configuration Reference
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `cosiallApiUrl` | `string` | **required** | Your Cosiall backend URL |
+| `cosiallApiKey` | `string` | **required** | Your Cosiall API key |
+| `proxyString` | `string` | `undefined` | Optional proxy (URL or colon format) |
+| `logLevel` | `string` | `'info'` | Log verbosity: `debug`, `info`, `warn`, `error` |
+| `profileCacheTtl` | `number` | `900000` | Profile cache TTL (15 minutes) |
+| `searchCacheTtl` | `number` | `900000` | Search cache TTL (15 minutes) |
+| `companyCacheTtl` | `number` | `600000` | Company cache TTL (10 minutes) |
+| `typeaheadCacheTtl` | `number` | `3600000` | Typeahead cache TTL (1 hour) |
+| `cookieRefreshInterval` | `number` | `900000` | How often to refresh cookie pool (15 min) |
+| `cookieFreshnessWindow` | `number` | `86400000` | Max age for cookies (24 hours) |
+| `maxRetries` | `number` | `2` | Retries on 5xx errors (same account) |
+| `retryDelayMs` | `number` | `700` | Delay between retries |
+| `accountCooldownMs` | `number` | `5000` | Cooldown after account rotation |
+| `maxFailuresBeforeCooldown` | `number` | `3` | Failures before triggering cooldown |
+| `maxRequestHistory` | `number` | `500` | Max requests in history log |
+| `eagerInitialization` | `boolean` | `true` | Fetch cookies during init (recommended) |
+| `fallbackWithoutProxyOnError` | `boolean` | `false` | Retry without proxy on 502/503/504 |
+| `onInitializationError` | `function` | `undefined` | Optional init error handler |
+| `sentryClient` | `SentryClient` | `undefined` | Optional Sentry integration |
+
+## Common Calls
+
+```ts
+import {
+  // Profiles
+  getProfileByVanity, getProfileByUrn, getProfilesBatch,
+  // Sales search + typeahead
+  searchSalesLeads, typeahead,
+  // Companies
+  resolveCompanyUniversalName, getCompanyById, getCompanyByUrl,
+  // Sales profile
+  getSalesNavigatorProfileDetails,
+  // Types (optional)
+  type LinkedInProfile, type SearchSalesResult, type Company,
+  type TypeaheadResult, type SalesNavigatorProfile,
+} from 'linkedin-secret-sauce';
+
+// Profiles
+const prof1: LinkedInProfile = await getProfileByVanity('satyanadella');
+const prof2: LinkedInProfile = await getProfileByUrn('urn:li:fsd_profile:ACwAAAbCdEf'); // URN or bare key
+const many = await getProfilesBatch(['a', 'b', 'c'], 4); // array aligned to inputs, nulls on failures
+
+// Sales search (returns { items, page } when options passed)
+const leads: SearchSalesResult = await searchSalesLeads('cto fintech london', { start: 0, count: 25 });
+// Note: on HTTP 400 with decoration "-14", the client automatically retries with "-17".
+
+// Typeahead
+const geo: TypeaheadResult = await typeahead({ type: 'BING_GEO', query: 'new', start: 0, count: 10 });
+
+// Static filters (years) - no API call needed, instant response
+import {
+  getYearsAtCompanyOptions,
+  getYearsInPositionOptions,
+  getYearsOfExperienceOptions,
+  YEARS_AT_COMPANY_OPTIONS,
+  YEARS_IN_POSITION_OPTIONS,
+  YEARS_OF_EXPERIENCE_OPTIONS,
+} from 'linkedin-secret-sauce';
+
+const yearsAtCompany: TypeaheadResult = await getYearsAtCompanyOptions();
+const yearsInPosition: TypeaheadResult = await getYearsInPositionOptions();
+const yearsOfExperience: TypeaheadResult = await getYearsOfExperienceOptions();
+// Returns: { items: [{ id: 1, text: "Less than 1 year", displayValue: "< 1 year" }, ...], page: { start: 0, count: 5, total: 5 } }
+
+// Or access constants directly (sync)
+const options = YEARS_AT_COMPANY_OPTIONS; // readonly array of { id, text, displayValue }
+
+// Companies
+const { companyId } = await resolveCompanyUniversalName('microsoft');
+const company: Company = await getCompanyById(companyId!);
+const same: Company = await getCompanyByUrl('https://www.linkedin.com/company/microsoft/');
+
+// Sales Navigator profile details
+const sales1: SalesNavigatorProfile =
+  await getSalesNavigatorProfileDetails('urn:li:fs_salesProfile:ACwAAA...');
+const sales2: SalesNavigatorProfile =
+  await getSalesNavigatorProfileDetails('profileId:ACwAAA...,authType:OUT_OF_NETWORK,authToken:xyz');
+// If the triple form returns 400 in your environment, the client automatically retries with the bare id.
+```
+
+## Email Enrichment
+
+The library includes a powerful email enrichment module that finds and verifies business emails using multiple providers in an optimized 3-phase strategy.
+
+```ts
+import { createEnrichmentClient } from 'linkedin-secret-sauce';
+
+const enrichment = createEnrichmentClient({
+  providers: {
+    // FREE providers (Phase 1 - parallel)
+    ldd: { apiUrl: process.env.LDD_API_URL, apiToken: process.env.LDD_API_TOKEN },
+    smartprospect: { email: process.env.SMARTLEAD_EMAIL, password: process.env.SMARTLEAD_PASSWORD },
+    trykitt: { apiKey: process.env.TRYKITT_API_KEY },
+    // construct and cosiall are enabled by default
+    
+    // Verification (Phase 2)
+    bounceban: { apiKey: process.env.BOUNCEBAN_API_KEY },
+    
+    // Paid finders (Phase 3 - only if needed)
+    hunter: { apiKey: process.env.HUNTER_API_KEY },
+    snovio: { clientId: process.env.SNOVIO_CLIENT_ID, clientSecret: process.env.SNOVIO_CLIENT_SECRET },
+  },
+});
+
+const result = await enrichment.enrich({
+  firstName: 'John',
+  lastName: 'Doe',
+  company: 'Acme Corp',
+  domain: 'acme.com',
+  linkedinUrl: 'https://linkedin.com/in/johndoe',
+});
+
+console.log(result.business_email);           // john.doe@acme.com
+console.log(result.business_email_source);    // 'trykitt'
+console.log(result.business_email_verified);  // true
+console.log(result.business_email_confidence); // 95
+```
+
+### Supported Enrichment Providers
+
+**FREE Providers:**
+- `construct` - Pattern guessing + MX verification (always enabled)
+- `cosiall` - Emails from LinkedIn profiles (uses global config)
+- `ldd` - LinkedIn Data Dump database (~500M verified emails)
+- `smartprospect` - SmartLead prospect database
+- `trykitt` - AI-powered email finder with catch-all verification
+
+**PAID Providers:**
+- `bounceban` - FREE single / $0.003 bulk - Catch-all specialist
+- `hunter` - $0.005/email - Domain search + email finder
+- `bouncer` - $0.006/email - SMTP verification (99%+ accuracy)
+- `dropcontact` - $0.01/email - Email finder API
+- `snovio` - $0.02/email - Email finding + verification
+
+For complete enrichment documentation, provider configuration, cost tracking, and advanced features, see **[ENRICHMENT.md](./ENRICHMENT.md)**.
+
+## Advanced Lead Search (filters)
+
+See `docs/SALES_SEARCH.md` for a full, strongly-typed `SalesSearchFilters` schema and examples.
+
+Quick preview:
+
+```ts
+const out = await searchSalesLeads('cto fintech', {
+  start: 0,
+  count: 25,
+  filters: {
+    // Titles allow id and free text
+    role: {
+      current_titles: { include: [{ id: 9 }, { text: 'Head of RevOps' }] },
+      seniority_ids: [320, 310],
+      functions: { include: ['ENGINEERING','IT'] },
+    },
+    personal: {
+      geography: { include: [{ type: 'country', value: 'United States', id: '103644278' }] },
+      profile_language: { include: ['en'] },
+      years_at_company_ids: [2,3],             // bucket ids 1..5
+      years_in_current_role_ids: [1,2,3],
+      years_experience_ids: [4,5],
+    },
+    company: {
+      current: { include: ['urn:li:organization:96151950'] }, // current company only
+      headcount: { include: ['51-200','201-500'] },            // or ['D','E']
+      type: { include: ['PUBLIC','PRIVATE'] },                 // or ['C','P']
+    },
+    spotlights: { posted_in_last_days: 30 },
+  },
+});
+```
+
+Notes
+- Relationship filters are not used in this package (customer-agnostic results).
+- Past company is intentionally not emitted — only CURRENT_COMPANY is supported.
+- Large facets (locations, industries, titles, functions): hydrate ids via typeahead.
+  - Example: `type: 'COMPANY_WITH_LIST'` to fetch company org URNs, then pass `urn:li:organization:ID`.
+  - `type: 'BING_GEO'` for person geography (REGION facet ids).
+- **Years filters** (years_at_company_ids, years_in_current_role_ids, years_experience_ids):
+  - Use the static helper functions `getYearsAtCompanyOptions()`, `getYearsInPositionOptions()`, `getYearsOfExperienceOptions()`
+  - These return predefined ranges (< 1 year, 1-2 years, 3-5 years, 6-10 years, 10+ years) with bucket ids 1-5
+  - **No API call needed** - instant response from in-memory constants
+  - Example: `years_at_company_ids: [2,3]` filters for "1-2 years" and "3-5 years" tenure
+
+Power users:
+- You can pass a complete `rawQuery` string to `searchSalesLeads` if you already have the full `query=(...)` payload; it takes precedence over `filters`.
+
+## Error Handling
+
+All throws are `LinkedInClientError` with fields `code`, `status?`, and `accountId?`.
+
+```ts
+import { LinkedInClientError } from 'linkedin-secret-sauce';
+
+try {
+  const res = await searchSalesLeads('cto');
+  // ...
+} catch (e) {
+  const err = e as LinkedInClientError;
+  switch (err.code) {
+    case 'NOT_FOUND':
+    case 'INVALID_INPUT':
+    case 'REQUEST_FAILED':
+    case 'ALL_ACCOUNTS_FAILED':
+    case 'NOT_INITIALIZED':
+    case 'INVALID_CONFIG':
+    case 'INITIALIZATION_FAILED':
+      // handle/log; check err.status and err.accountId
+      break;
+  }
+}
+```
+
+## Metrics and Request History
+
+```ts
+import { getSnapshot, getRequestHistory, clearRequestHistory } from 'linkedin-secret-sauce';
+
+// Metrics (counters for requests, retries, cache hits, etc.)
+const snap = getSnapshot();      // { profileFetches, httpSuccess, httpFailures, ... }
+
+// Request history (bounded in-memory log; secrets redacted)
+const recent = getRequestHistory();
+clearRequestHistory();
+```
+
+## Account Pool & Proxy
+
+- Accounts (cookies) are fetched from your Cosiall backend (`/api/flexiq/linkedin-cookies/all`, authorized with `COSIALL_API_KEY`).
+- The pool round-robins across accounts, rotates on 401/403/429 (auth/rate limit) and applies cooldowns, and periodically refreshes.
+- With `eagerInitialization: true` (default), cookies are fetched during `initializeLinkedInClient()` for faster first requests.
+- Proxy: pass `proxyString` in config.
+  - Accepted forms:
+    - URL: `http://user:pass@host:port` (IPv6 `[::1]` supported)
+    - Colon: `host:port[:user:pass]`
+  - The client sets `HTTP_PROXY`/`HTTPS_PROXY` during the request and restores afterwards.
+  - If `fallbackWithoutProxyOnError: true`, the client retries without proxy on 502/503/504 errors.
+
+## Minimal Express Route Example
+
+```ts
+import express from 'express';
+import { initLinkedIn } from './bootstrap/linkedin'; // your wrapper above
+import {
+  getProfileByVanity, searchSalesLeads, getCompanyById,
+  typeahead, getSalesNavigatorProfileDetails,
+  createEnrichmentClient,
+} from 'linkedin-secret-sauce';
+
+await initLinkedIn();
+
+// Optional: Set up email enrichment
+const enrichment = createEnrichmentClient({
+  providers: {
+    trykitt: { apiKey: process.env.TRYKITT_API_KEY },
+    bounceban: { apiKey: process.env.BOUNCEBAN_API_KEY },
+    hunter: { apiKey: process.env.HUNTER_API_KEY },
+  },
+});
+
+const app = express();
+app.use(express.json());
+
+app.post('/api/profile/vanity', async (req, res) => {
+  const { vanity } = req.body;
+  res.json(await getProfileByVanity(vanity));
+});
+
+app.post('/api/search', async (req, res) => {
+  const { keywords, start = 0, count = 25 } = req.body;
+  res.json(await searchSalesLeads(keywords, { start, count }));
+});
+
+app.post('/api/company/by-id', async (req, res) => {
+  const { id } = req.body;
+  res.json(await getCompanyById(String(id)));
+});
+
+app.post('/api/typeahead', async (req, res) => {
+  const { type, query, start = 0, count = 10 } = req.body;
+  res.json(await typeahead({ type, query, start, count }));
+});
+
+app.post('/api/sales/profile', async (req, res) => {
+  const { idOrUrn } = req.body; // supports bare id, URN, or triple
+  res.json(await getSalesNavigatorProfileDetails(idOrUrn));
+});
+
+app.post('/api/enrich-email', async (req, res) => {
+  const { firstName, lastName, domain, company } = req.body;
+  const result = await enrichment.enrich({ firstName, lastName, domain, company });
+  res.json(result);
+});
+
+app.listen(3000, () => console.log('API listening on :3000'));
+```
+
+## Behavior Notes
+
+- Caches: in-process memory with TTLs; stale entries evicted on access.
+- Retries: 5xx retried on same account (`maxRetries`) with `retryDelayMs` backoff.
+- Rotations: 401/403/429 trigger account rotation + cooldown.
+- Logging: JSON lines to stdout with secrets masked (`cookie`, `authorization`, etc.).
+- Initialization: With `eagerInitialization: true` (default), cookies are fetched during init for faster first requests.
+
+## Troubleshooting
+
+- 400 on Sales Profile:
+  - The library auto-retries triple → bare id when 400 occurs.
+  - You can pass either a bare profile id/URN or the triple `(profileId:...,authType:...,authToken:...)`.
+- Timeouts:
+  - Node 18's `fetch` has no default timeout; if you need a hard timeout, wrap calls with `AbortController` at the app level (or open an issue to add a built-in timeout).
+- Initialization Failures:
+  - If `eagerInitialization: true` and cookie fetch fails, the client throws by default.
+  - Use `onInitializationError` callback to handle gracefully instead of throwing.
+  - Or set `eagerInitialization: false` to defer cookie fetching until first API call.
+- Proxy Errors (502/503/504):
+  - Set `fallbackWithoutProxyOnError: true` to automatically retry without proxy on gateway errors.
+
+## CommonJS
+
+If you're using CJS, use `require`:
+
+```js
+const { initializeLinkedInClient, getProfileByVanity, createEnrichmentClient } = require('linkedin-secret-sauce');
+```