deepline 0.1.74 → 0.1.77

package/dist/repo/shared_libs/play-runtime/cell-staleness.ts

@@ -0,0 +1,88 @@
+export type CellStalenessPolicy = {
+  staleAfterSeconds?: number;
+};
+
+export type CellStalenessMeta = {
+  status?: string | null;
+  completedAt?: number | null;
+};
+
+export type CellStalenessDecision =
+  | { action: 'recompute'; reason: 'missing' | 'failed' | 'stale' }
+  | { action: 'reuse'; reason: 'fresh' | 'no_policy' };
+
+export type CellStalenessPolicyByField = Record<string, CellStalenessPolicy>;
+
+export const DEEPLINE_CELL_META_FIELD = '__deeplineCellMeta';
+
+export function validateStaleAfterSeconds(
+  staleAfterSeconds: number | undefined,
+  label = 'staleAfterSeconds',
+): void {
+  if (staleAfterSeconds === undefined) {
+    return;
+  }
+  if (
+    !Number.isFinite(staleAfterSeconds) ||
+    !Number.isInteger(staleAfterSeconds) ||
+    staleAfterSeconds <= 0
+  ) {
+    throw new Error(`${label} must be a positive whole number of seconds.`);
+  }
+}
+
+export function normalizeCellStalenessPolicy(
+  policy: CellStalenessPolicy | undefined,
+): CellStalenessPolicy {
+  validateStaleAfterSeconds(policy?.staleAfterSeconds);
+  return policy?.staleAfterSeconds === undefined
+    ? {}
+    : { staleAfterSeconds: policy.staleAfterSeconds };
+}
+
+export function shouldRecomputeCell(input: {
+  hasValue: boolean;
+  meta?: CellStalenessMeta | null;
+  policy?: CellStalenessPolicy;
+  nowMs?: number;
+}): CellStalenessDecision {
+  if (!input.hasValue) {
+    return { action: 'recompute', reason: 'missing' };
+  }
+
+  const status = String(input.meta?.status ?? '').trim();
+  if (status === 'failed') {
+    return { action: 'recompute', reason: 'failed' };
+  }
+
+  const staleAfterSeconds = input.policy?.staleAfterSeconds;
+  validateStaleAfterSeconds(staleAfterSeconds);
+  if (staleAfterSeconds === undefined) {
+    return { action: 'reuse', reason: 'no_policy' };
+  }
+
+  const completedAt =
+    typeof input.meta?.completedAt === 'number' &&
+    Number.isFinite(input.meta.completedAt)
+      ? input.meta.completedAt
+      : null;
+  if (completedAt === null) {
+    return { action: 'recompute', reason: 'missing' };
+  }
+
+  const ageMs = Math.max(0, (input.nowMs ?? Date.now()) - completedAt);
+  return ageMs > staleAfterSeconds * 1000
+    ? { action: 'recompute', reason: 'stale' }
+    : { action: 'reuse', reason: 'fresh' };
+}
+
+export function cellPolicyFields(
+  policies: CellStalenessPolicyByField | undefined,
+): string[] {
+  if (!policies) {
+    return [];
+  }
+  return Object.entries(policies)
+    .filter(([, policy]) => policy.staleAfterSeconds !== undefined)
+    .map(([field]) => field);
+}

package/dist/repo/shared_libs/play-runtime/email-status.ts

@@ -0,0 +1,301 @@
+export type EmailStatusVerdict =
+  | 'send'
+  | 'send_with_caution'
+  | 'verify_next'
+  | 'hold'
+  | 'drop';
+
+export type EmailStatusValue =
+  | 'valid'
+  | 'invalid'
+  | 'catch_all'
+  | 'valid_catch_all'
+  | 'unknown'
+  | 'do_not_mail'
+  | 'spamtrap'
+  | 'abuse'
+  | 'disposable';
+
+export type EmailDeliverability = 'high' | 'medium' | 'low' | 'unknown';
+
+export type EmailMxClass =
+  | 'consumer_mailbox'
+  | 'workspace_mailbox'
+  | 'security_gateway'
+  | 'on_prem'
+  | 'unknown';
+
+export type EmailStatus = {
+  verdict: EmailStatusVerdict;
+  status: EmailStatusValue;
+  verified: boolean;
+  confidence: number | null;
+  reasons: string[];
+  signals: {
+    catch_all: boolean | null;
+    deliverability: EmailDeliverability;
+    mx_class: EmailMxClass;
+    mx_provider: string | null;
+    mx_record: string | null;
+    fraud_score: number | null;
+    disposable: boolean | null;
+    role_based: boolean | null;
+    free_email: boolean | null;
+    abuse: boolean | null;
+    spamtrap: boolean | null;
+    suspect: boolean | null;
+    valid: boolean | null;
+  };
+  provider: {
+    name: string;
+    raw_status: string | boolean | number | null;
+    raw_score: number | null;
+  };
+};
+
+export type EmailStatusMapEntry = {
+  status: EmailStatusValue;
+  verdict?: EmailStatusVerdict;
+  verified?: boolean;
+  reason?: string;
+};
+
+export type EmailStatusRule = EmailStatusMapEntry & {
+  when: Record<string, string | number | boolean | null>;
+};
+
+export type EmailStatusExtractorConfig = {
+  provider: string;
+  rawStatus?: string[];
+  rawScore?: string[];
+  valid?: string[];
+  deliverability?: string[];
+  catchAll?: string[];
+  mxProvider?: string[];
+  mxRecord?: string[];
+  fraudScore?: string[];
+  disposable?: string[];
+  roleBased?: string[];
+  freeEmail?: string[];
+  abuse?: string[];
+  spamtrap?: string[];
+  suspect?: string[];
+  statusMap?: Record<string, EmailStatusMapEntry>;
+  rules?: EmailStatusRule[];
+};
+
+export type EmailStatusBuildInput = {
+  config: EmailStatusExtractorConfig;
+  values: Record<string, unknown>;
+};
+
+const DEFAULT_STATUS_MAP: Record<string, EmailStatusMapEntry> = {
+  verified: { status: 'valid', verdict: 'send', verified: true },
+  valid: { status: 'valid', verdict: 'send', verified: true },
+  deliverable: { status: 'valid', verdict: 'send', verified: true },
+  true: { status: 'valid', verdict: 'send', verified: true },
+  invalid: { status: 'invalid', verdict: 'drop', verified: false },
+  undeliverable: { status: 'invalid', verdict: 'drop', verified: false },
+  false: { status: 'invalid', verdict: 'drop', verified: false },
+  'catch-all': {
+    status: 'catch_all',
+    verdict: 'verify_next',
+    verified: false,
+  },
+  catch_all: {
+    status: 'catch_all',
+    verdict: 'verify_next',
+    verified: false,
+  },
+  valid_catch_all: {
+    status: 'valid_catch_all',
+    verdict: 'send_with_caution',
+    verified: true,
+  },
+  accept_all: {
+    status: 'catch_all',
+    verdict: 'verify_next',
+    verified: false,
+  },
+  unknown: { status: 'unknown', verdict: 'hold', verified: false },
+  unavailable: { status: 'unknown', verdict: 'hold', verified: false },
+  do_not_mail: { status: 'do_not_mail', verdict: 'drop', verified: false },
+  spamtrap: { status: 'spamtrap', verdict: 'drop', verified: false },
+  abuse: { status: 'abuse', verdict: 'drop', verified: false },
+  disposable: { status: 'disposable', verdict: 'drop', verified: false },
+};
+
+function normalizeKey(value: unknown): string | null {
+  if (value == null) return null;
+  if (typeof value === 'boolean') return String(value);
+  const normalized = String(value).trim().toLowerCase().replace(/\s+/g, '_');
+  return normalized || null;
+}
+
+function boolish(value: unknown): boolean | null {
+  if (typeof value === 'boolean') return value;
+  if (typeof value === 'number') return value === 1 ? true : value === 0 ? false : null;
+  if (typeof value !== 'string') return null;
+  const normalized = value.trim().toLowerCase();
+  if (['true', 'yes', 'y', '1'].includes(normalized)) return true;
+  if (['false', 'no', 'n', '0'].includes(normalized)) return false;
+  return null;
+}
+
+function numberish(value: unknown): number | null {
+  if (typeof value === 'number' && Number.isFinite(value)) return value;
+  if (typeof value !== 'string' || value.trim() === '') return null;
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+function stringish(value: unknown): string | null {
+  return typeof value === 'string' && value.trim() ? value.trim() : null;
+}
+
+function deliverability(value: unknown): EmailDeliverability {
+  const normalized = normalizeKey(value);
+  return normalized === 'high' || normalized === 'medium' || normalized === 'low'
+    ? normalized
+    : 'unknown';
+}
+
+function mxClass(mxProvider: unknown, mxRecord: unknown): EmailMxClass {
+  const haystack = `${stringish(mxProvider) ?? ''} ${stringish(mxRecord) ?? ''}`.toLowerCase();
+  if (!haystack.trim()) return 'unknown';
+  if (
+    /proofpoint|pphosted|mimecast|barracuda|ess\.barracudanetworks|ironport|cisco|iphmx|messagelabs|symantec/.test(
+      haystack,
+    )
+  ) {
+    return 'security_gateway';
+  }
+  if (/aspmx\.l\.google|google|g-suite|google workspace/.test(haystack)) {
+    return 'workspace_mailbox';
+  }
+  if (/protection\.outlook|office365|microsoft|outlook|exchange online/.test(haystack)) {
+    return 'workspace_mailbox';
+  }
+  if (/gmail|yahoo|icloud|aol|hotmail/.test(haystack)) return 'consumer_mailbox';
+  if (/postfix|exim|sendmail|zimbra|plesk|cpanel|mail\./.test(haystack)) return 'on_prem';
+  return 'unknown';
+}
+
+function entryForStatus(
+  key: string | null,
+  map: Record<string, EmailStatusMapEntry> | undefined,
+): EmailStatusMapEntry | null {
+  if (!key) return null;
+  return map?.[key] ?? DEFAULT_STATUS_MAP[key] ?? null;
+}
+
+function read(values: Record<string, unknown>, name: string): unknown {
+  return values[name];
+}
+
+function matchesRule(
+  rule: EmailStatusRule,
+  values: Record<string, unknown>,
+): boolean {
+  return Object.entries(rule.when).every(([key, expected]) => {
+    const actual = read(values, key);
+    if (key.endsWith('Lt')) {
+      const source = numberish(read(values, key.slice(0, -2)));
+      return typeof expected === 'number' && source != null && source < expected;
+    }
+    if (typeof expected === 'boolean') return boolish(actual) === expected;
+    if (typeof expected === 'number') return numberish(actual) === expected;
+    return normalizeKey(actual) === normalizeKey(expected);
+  });
+}
+
+export function buildEmailStatus({
+  config,
+  values,
+}: EmailStatusBuildInput): EmailStatus {
+  const rawStatus = read(values, 'rawStatus');
+  const rawScore = numberish(read(values, 'rawScore'));
+  const valid = boolish(read(values, 'valid'));
+  const catchAll = boolish(read(values, 'catchAll'));
+  const disposable = boolish(read(values, 'disposable'));
+  const abuse = boolish(read(values, 'abuse'));
+  const spamtrap = boolish(read(values, 'spamtrap'));
+  const suspect = boolish(read(values, 'suspect'));
+  const rawKey = normalizeKey(rawStatus);
+  const mapped =
+    config.rules?.find((rule) => matchesRule(rule, values)) ??
+    entryForStatus(rawKey, config.statusMap) ??
+    entryForStatus(valid == null ? null : String(valid), config.statusMap);
+
+  const status =
+    mapped?.status ??
+    (disposable
+      ? 'disposable'
+      : abuse
+        ? 'abuse'
+        : spamtrap
+          ? 'spamtrap'
+          : catchAll
+            ? 'catch_all'
+            : valid === true
+              ? 'valid'
+              : valid === false
+                ? 'invalid'
+                : 'unknown');
+  const defaultVerdict: EmailStatusVerdict =
+    status === 'valid'
+      ? 'send'
+      : status === 'valid_catch_all'
+        ? 'send_with_caution'
+        : status === 'catch_all'
+          ? 'verify_next'
+          : status === 'unknown'
+            ? 'hold'
+            : 'drop';
+  const verdict = mapped?.verdict ?? defaultVerdict;
+  const verified =
+    mapped?.verified ??
+    (status === 'valid' || status === 'valid_catch_all' || verdict === 'send');
+  const reasons = [
+    mapped?.reason,
+    catchAll ? 'catch_all_domain' : null,
+    mxClass(read(values, 'mxProvider'), read(values, 'mxRecord')) ===
+    'security_gateway'
+      ? 'security_gateway_mx'
+      : null,
+    suspect ? 'provider_marked_suspect' : null,
+  ].filter((reason): reason is string => typeof reason === 'string');
+
+  return {
+    verdict,
+    status,
+    verified,
+    confidence: rawScore,
+    reasons,
+    signals: {
+      catch_all: catchAll,
+      deliverability: deliverability(read(values, 'deliverability')),
+      mx_class: mxClass(read(values, 'mxProvider'), read(values, 'mxRecord')),
+      mx_provider: stringish(read(values, 'mxProvider')),
+      mx_record: stringish(read(values, 'mxRecord')),
+      fraud_score: numberish(read(values, 'fraudScore')),
+      disposable,
+      role_based: boolish(read(values, 'roleBased')),
+      free_email: boolish(read(values, 'freeEmail')),
+      abuse,
+      spamtrap,
+      suspect,
+      valid,
+    },
+    provider: {
+      name: config.provider,
+      raw_status:
+        typeof rawStatus === 'string' ||
+        typeof rawStatus === 'boolean' ||
+        typeof rawStatus === 'number'
+          ? rawStatus
+          : null,
+      raw_score: rawScore,
+    },
+  };
+}

package/dist/repo/shared_libs/play-runtime/step-program-dataset-builder.ts

@@ -3,11 +3,13 @@ export type StepProgramDatasetOptions = {
     row: Record<string, unknown>,
     index: number,
   ) => boolean | Promise<boolean>;
+  staleAfterSeconds?: number;
 };
 
 export type StepProgramDatasetStep<TResolver> = {
   name: string;
   resolver: TResolver;
+  staleAfterSeconds?: number;
 };
 
 export type StepProgramDatasetProgram<TStep> = {
@@ -84,6 +86,9 @@ export class StepProgramDatasetBuilder<
       {
         name,
         resolver: this.applyDerivationOptions(resolver, options),
+        ...(options?.staleAfterSeconds !== undefined
+          ? { staleAfterSeconds: options.staleAfterSeconds }
+          : {}),
       } as TStep,
     ];
     return this;

package/dist/repo/shared_libs/play-runtime/tool-result.ts

@@ -9,7 +9,15 @@ export type {
   ToolResultTargetMetadata,
   ToolResultTargetAccessor,
 } from './tool-result-types';
-
+export type {
+  EmailDeliverability,
+  EmailMxClass,
+  EmailStatus,
+  EmailStatusValue,
+  EmailStatusVerdict,
+} from './email-status';
+
+import { buildEmailStatus } from './email-status';
 import type {
   SerializedToolExecuteResult,
   ToolExecuteResult,
@@ -302,6 +310,50 @@ function findFirstTargetByPath(
   return null;
 }
 
+function firstValueForPaths(
+  result: unknown,
+  paths: readonly string[] | undefined,
+): ToolResultTargetMetadata | null {
+  return findFirstTargetByPath(result, paths);
+}
+
+function buildEmailStatusTarget(
+  result: unknown,
+  descriptor: ToolResultExtractorDescriptor,
+): ToolResultTargetMetadata | null {
+  const config = descriptor.emailStatus;
+  if (!config) return null;
+  const values: Record<string, unknown> = {};
+  const pathSets: Record<string, readonly string[] | undefined> = {
+    rawStatus: config.rawStatus,
+    rawScore: config.rawScore,
+    valid: config.valid,
+    deliverability: config.deliverability,
+    catchAll: config.catchAll,
+    mxProvider: config.mxProvider,
+    mxRecord: config.mxRecord,
+    fraudScore: config.fraudScore,
+    disposable: config.disposable,
+    roleBased: config.roleBased,
+    freeEmail: config.freeEmail,
+    abuse: config.abuse,
+    spamtrap: config.spamtrap,
+    suspect: config.suspect,
+  };
+  let firstPath: string | null = null;
+  for (const [name, paths] of Object.entries(pathSets)) {
+    const match = firstValueForPaths(result, paths);
+    if (!match) continue;
+    values[name] = match.value;
+    firstPath ??= match.path;
+  }
+  if (!firstPath) return null;
+  return {
+    path: firstPath,
+    value: buildEmailStatus({ config, values }),
+  };
+}
+
 function findFirstTargetByKey(
   result: unknown,
   target: string,
@@ -507,6 +559,11 @@ function buildTargets(
 ): Record<string, ToolResultTargetMetadata> {
   const targets: Record<string, ToolResultTargetMetadata> = {};
   for (const [target, descriptor] of Object.entries(extractors ?? {})) {
+    const emailStatusTarget = buildEmailStatusTarget(result, descriptor);
+    if (emailStatusTarget) {
+      targets[target] = emailStatusTarget;
+      continue;
+    }
     const fromExtractor = findFirstTargetByPath(result, descriptor.paths);
     if (!fromExtractor) continue;
     const transformed = coerceToEnum(

package/dist/repo/shared_libs/plays/row-identity.ts

@@ -192,46 +192,6 @@ export function normalizeTableNamespace(value: string): string {
   );
 }
 
-export function resolveStaleMapTableNamespace(
-  mapKey: string,
-  staleAfterSeconds?: number | null,
-  nowMs: number = Date.now(),
-): string {
-  const normalizedMapKey = normalizeTableNamespace(mapKey);
-  if (staleAfterSeconds === undefined || staleAfterSeconds === null) {
-    return normalizedMapKey;
-  }
-
-  if (
-    !Number.isFinite(staleAfterSeconds) ||
-    !Number.isInteger(staleAfterSeconds) ||
-    staleAfterSeconds <= 0
-  ) {
-    throw new Error(
-      'ctx.dataset() staleAfterSeconds must be a positive whole number of seconds.',
-    );
-  }
-
-  const bucket = Math.floor(nowMs / (staleAfterSeconds * 1000));
-  const stalePartitionKey = `stale_${staleAfterSeconds}_${bucket}`;
-  const candidate = `${normalizedMapKey}_${stalePartitionKey}`;
-  if (candidate.length <= MAP_KEY_NAMESPACE_MAX_LENGTH) {
-    return candidate;
-  }
-
-  const digest = sha256Hex(`${normalizedMapKey}\n${stalePartitionKey}`).slice(
-    0,
-    12,
-  );
-  const prefixLength = Math.max(
-    1,
-    MAP_KEY_NAMESPACE_MAX_LENGTH - digest.length - 1,
-  );
-  const prefix =
-    normalizedMapKey.slice(0, prefixLength).replace(/_+$/g, '') || 'map';
-  return `${prefix}_${digest}`;
-}
-
 export function validatePlaySheetTableName(
   playName: string,
   tableNamespace: string,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.74",
+  "version": "0.1.77",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {