owletto 1.0.0

package/src/connector-types.ts

@@ -0,0 +1,299 @@
+/**
+ * Connector Types
+ *
+ * Type definitions for the V1 integration platform.
+ * Defines the contract between connectors, the runtime, and the platform.
+ */
+
+// =============================================================================
+// Connector Definition
+// =============================================================================
+
+export interface ConnectorDefinition {
+  /** Unique connector key, e.g. 'google.gmail' */
+  key: string;
+  /** Human-readable name */
+  name: string;
+  /** Description of what this connector does */
+  description?: string;
+  /** Semantic version */
+  version: string;
+  /** Auth configuration */
+  authSchema?: ConnectorAuthSchema;
+  /** Available feed definitions (keyed by feed_key) */
+  feeds?: Record<string, FeedDefinition>;
+  /** Available action definitions (keyed by action_key) */
+  actions?: Record<string, ActionDefinition>;
+  /** Global connector options schema (JSON Schema) */
+  optionsSchema?: Record<string, unknown>;
+}
+
+// =============================================================================
+// Auth
+// =============================================================================
+
+export interface ConnectorAuthSchema {
+  methods: ConnectorAuthMethod[];
+}
+
+export type ConnectorAuthMethod = ConnectorAuthNone | ConnectorAuthEnvKeys | ConnectorAuthOAuth;
+
+export interface ConnectorAuthNone {
+  type: 'none';
+}
+
+export interface ConnectorAuthEnvKeys {
+  type: 'env_keys';
+  required?: boolean;
+  fields: Array<{
+    key: string;
+    label?: string;
+    description?: string;
+    example?: string;
+    secret?: boolean;
+  }>;
+  description?: string;
+}
+
+export interface ConnectorAuthOAuth {
+  type: 'oauth';
+  provider: string;
+  requiredScopes: string[];
+  required?: boolean;
+  description?: string;
+}
+
+// =============================================================================
+// Feed Definition
+// =============================================================================
+
+export interface FeedDefinition {
+  /** Feed key, e.g. 'threads' */
+  key: string;
+  /** Human-readable name */
+  name: string;
+  /** Description */
+  description?: string;
+  /** JSON Schema for feed-specific config */
+  configSchema?: Record<string, unknown>;
+}
+
+export enum FeedMode {
+  /** Connector code runs on worker, syncs data */
+  sync = 'sync',
+  /** Virtual feed backed by saved queries (future) */
+  virtual = 'virtual',
+}
+
+// =============================================================================
+// Action Definition
+// =============================================================================
+
+export interface ActionDefinition {
+  /** Action key, e.g. 'draft_email' */
+  key: string;
+  /** Human-readable name */
+  name: string;
+  /** Description */
+  description?: string;
+  /** Whether this action requires human approval before execution */
+  requiresApproval: boolean;
+  /** MCP tool annotations for client-side confirmation UX */
+  annotations?: {
+    destructiveHint?: boolean;
+    openWorldHint?: boolean;
+    idempotentHint?: boolean;
+  };
+  /** JSON Schema for action input */
+  inputSchema?: Record<string, unknown>;
+  /** JSON Schema for action output */
+  outputSchema?: Record<string, unknown>;
+}
+
+// =============================================================================
+// Connection
+// =============================================================================
+
+export interface Connection {
+  id: number;
+  organizationId: string;
+  connectorKey: string;
+  displayName?: string;
+  status: 'active' | 'paused' | 'error' | 'revoked';
+  accountId?: string;
+  credentials?: Record<string, unknown>;
+  entityIds?: number[];
+  config?: Record<string, unknown>;
+  errorMessage?: string;
+  createdBy?: string;
+  createdAt: Date;
+  updatedAt: Date;
+}
+
+// =============================================================================
+// Feed
+// =============================================================================
+
+export interface Feed {
+  id: number;
+  organizationId: string;
+  connectionId: number;
+  feedKey: string;
+  status: 'active' | 'paused' | 'error';
+  entityIds?: number[];
+  config?: Record<string, unknown>;
+  checkpoint?: Record<string, unknown>;
+  syncIntervalMs?: number;
+  nextSyncAt?: Date;
+  lastSyncAt?: Date;
+  lastSyncStatus?: string;
+  lastError?: string;
+  consecutiveFailures: number;
+  itemsCollected: number;
+  createdAt: Date;
+  updatedAt: Date;
+}
+
+// =============================================================================
+// Run
+// =============================================================================
+
+export type RunType = 'sync' | 'action' | 'code' | 'insight';
+export type RunStatus =
+  | 'pending'
+  | 'claimed'
+  | 'running'
+  | 'completed'
+  | 'failed'
+  | 'cancelled'
+  | 'timeout';
+export type ApprovalStatus = 'pending' | 'approved' | 'rejected' | 'auto';
+
+export interface Run {
+  id: number;
+  organizationId: string;
+  runType: RunType;
+  feedId?: number;
+  connectionId?: number;
+  actionKey?: string;
+  actionInput?: Record<string, unknown>;
+  actionOutput?: Record<string, unknown>;
+  approvalStatus: ApprovalStatus;
+  status: RunStatus;
+  claimedBy?: string;
+  claimedAt?: Date;
+  lastHeartbeatAt?: Date;
+  completedAt?: Date;
+  connectorKey?: string;
+  connectorVersion?: string;
+  checkpoint?: Record<string, unknown>;
+  itemsCollected: number;
+  errorMessage?: string;
+  createdAt: Date;
+}
+
+// =============================================================================
+// Event Envelope
+// =============================================================================
+
+/**
+ * EventEnvelope is the standard output format for connector sync operations.
+ * Each envelope becomes a row in the events table.
+ */
+export interface EventEnvelope {
+  /** Platform's unique ID for this item */
+  external_id: string;
+  /** Main text content */
+  content: string;
+  /** Title / subject line */
+  title?: string;
+  /** Author name or email */
+  author?: string;
+  /** Link to original content */
+  url?: string;
+  /** When the content was originally created/published */
+  published_at: Date;
+  /** Content kind (e.g. 'thread', 'message', 'email', 'issue') */
+  kind?: string;
+  /** Engagement/relevance score (0-100) */
+  score?: number;
+  /** Parent reference for hierarchical content */
+  parent_external_id?: string;
+  /** Arbitrary metadata */
+  metadata?: Record<string, unknown>;
+  /** Pre-computed embedding vector */
+  embedding?: number[];
+}
+
+// =============================================================================
+// Sync Context & Result
+// =============================================================================
+
+/**
+ * Context passed to ConnectorRuntime.sync()
+ */
+export interface SyncContext {
+  /** Feed configuration */
+  feedKey: string;
+  config: Record<string, unknown>;
+  /** Previous checkpoint (null on first sync) */
+  checkpoint: Record<string, unknown> | null;
+  /** OAuth credentials (if applicable) */
+  credentials: SyncCredentials | null;
+  /** Entity IDs this feed is linked to */
+  entityIds: number[];
+}
+
+export interface SyncCredentials {
+  provider: string;
+  accessToken: string;
+  refreshToken?: string | null;
+  expiresAt?: string | null;
+  scope?: string | null;
+}
+
+/**
+ * Result from ConnectorRuntime.sync()
+ */
+export interface SyncResult {
+  /** Events to write to the events table */
+  events: EventEnvelope[];
+  /** Updated checkpoint to persist */
+  checkpoint: Record<string, unknown> | null;
+  /** Optional metadata about the sync */
+  metadata?: {
+    items_found?: number;
+    items_skipped?: number;
+    [key: string]: unknown;
+  };
+}
+
+// =============================================================================
+// Action Context & Result
+// =============================================================================
+
+/**
+ * Context passed to ConnectorRuntime.execute()
+ */
+export interface ActionContext {
+  /** Action key to execute */
+  actionKey: string;
+  /** Action input parameters */
+  input: Record<string, unknown>;
+  /** OAuth credentials (if applicable) */
+  credentials: SyncCredentials | null;
+  /** Connection config */
+  config: Record<string, unknown>;
+}
+
+/**
+ * Result from ConnectorRuntime.execute()
+ */
+export interface ActionResult {
+  /** Whether the action succeeded */
+  success: boolean;
+  /** Output data */
+  output?: Record<string, unknown>;
+  /** Error message if failed */
+  error?: string;
+}

package/src/http.ts

@@ -0,0 +1,82 @@
+import ky, { type KyInstance, type Options } from 'ky';
+
+/**
+ * Shared HTTP client configuration for all crawlers
+ */
+
+/**
+ * Default retry configuration
+ * - Max 2 retries (3 total attempts)
+ * - Only retry transient errors (429, 5xx)
+ * - Exponential backoff up to 5 seconds
+ * - 30s timeout per request
+ */
+const defaultRetryConfig = {
+  retry: {
+    limit: 2, // Max 2 retries (3 total attempts)
+    methods: ['get', 'post'],
+    statusCodes: [
+      408, // Request Timeout
+      429, // Too Many Requests (rate limit)
+      500, // Internal Server Error
+      502, // Bad Gateway
+      503, // Service Unavailable
+      504, // Gateway Timeout
+    ],
+    backoffLimit: 5000, // Max 5 seconds delay between retries
+  },
+  timeout: 30000, // 30 second timeout per request
+};
+
+/**
+ * Create a configured ky instance with custom options
+ */
+export function createHttpClient(options?: Options): KyInstance {
+  return ky.create({
+    ...defaultRetryConfig,
+    ...options,
+    // Merge retry config if provided
+    retry: options?.retry
+      ? {
+          ...defaultRetryConfig.retry,
+          ...(typeof options.retry === 'number' ? { limit: options.retry } : options.retry),
+        }
+      : defaultRetryConfig.retry,
+  });
+}
+
+/**
+ * Default HTTP client for crawlers with standard User-Agent
+ */
+export const httpClient = createHttpClient({
+  headers: {
+    'User-Agent': 'UserResearchBot/1.0',
+  },
+});
+
+/**
+ * Create an HTTP client with authentication headers
+ */
+export function createAuthenticatedClient(
+  authHeader: string,
+  additionalHeaders?: Record<string, string>
+): KyInstance {
+  return createHttpClient({
+    headers: {
+      'User-Agent': 'UserResearchBot/1.0',
+      Authorization: authHeader,
+      ...additionalHeaders,
+    },
+  });
+}
+
+/**
+ * HTTP client for JSON APIs
+ */
+export const jsonHttpClient = createHttpClient({
+  headers: {
+    'User-Agent': 'UserResearchBot/1.0',
+    Accept: 'application/json',
+    'Content-Type': 'application/json',
+  },
+});

package/src/index.ts

@@ -0,0 +1,106 @@
+// =============================================================================
+// V1 Integration Platform — Connector SDK
+// =============================================================================
+
+// TypeBox (schema authoring convenience)
+export type { Static } from '@sinclair/typebox';
+export { Type } from '@sinclair/typebox';
+// ky (shared HTTP dependency)
+export type { KyInstance, Options } from 'ky';
+export { default as ky, HTTPError } from 'ky';
+// Connector runtime & types (primary API)
+export { ConnectorRuntime } from './connector-runtime.js';
+export type {
+  ActionContext,
+  ActionDefinition,
+  ActionResult,
+  ApprovalStatus,
+  Connection,
+  ConnectorAuthEnvKeys,
+  ConnectorAuthMethod,
+  ConnectorAuthNone,
+  ConnectorAuthOAuth,
+  ConnectorAuthSchema,
+  ConnectorDefinition,
+  EventEnvelope,
+  Feed,
+  FeedDefinition,
+  FeedMode,
+  Run,
+  RunStatus,
+  RunType,
+  SyncContext,
+  SyncCredentials,
+  SyncResult,
+} from './connector-types.js';
+
+// HTTP clients
+export {
+  createAuthenticatedClient,
+  createHttpClient,
+  httpClient,
+  jsonHttpClient,
+} from './http.js';
+
+// Logger
+export { sdkLogger, sdkLogger as logger } from './logger.js';
+
+// Retry
+export { withHttpRetry } from './retry.js';
+
+// Scoring
+export { calculateEngagementScore } from './scoring.js';
+
+// =============================================================================
+// Legacy Crawler SDK (kept for backward compatibility during migration)
+// =============================================================================
+
+export type { ApiSessionState } from './api-paginated.js';
+export { ApiPaginatedCrawler } from './api-paginated.js';
+export { BaseCrawler, RateLimitError } from './base.js';
+export type { BrowserLaunchOptions, EnhancedBrowser } from './browser/launcher.js';
+export {
+  captureErrorArtifacts,
+  launchBrowser,
+  withErrorCapture,
+} from './browser/launcher.js';
+export type { StealthBrowser, StealthBrowserOptions } from './browser/stealth.js';
+export {
+  getRandomDelay,
+  humanWait,
+  launchStealthBrowser,
+  randomScroll,
+  testBotDetection,
+} from './browser/stealth.js';
+export type {
+  BrowserCrawlerConfig,
+  BrowserPaginationConfig,
+  BrowserSessionState,
+  CaptchaConfig,
+  CookieConsentConfig,
+} from './browser-paginated.js';
+export { BrowserPaginatedCrawler } from './browser-paginated.js';
+export type {
+  PageFetchResult,
+  PaginatedCheckpoint,
+  PaginateResult,
+  PaginationConfig,
+} from './paginated.js';
+export { PaginatedCrawler } from './paginated.js';
+export type {
+  Checkpoint,
+  Content,
+  CrawlerAuthEnvField,
+  CrawlerAuthEnvKeysMethod,
+  CrawlerAuthMethod,
+  CrawlerAuthOAuthMethod,
+  CrawlerAuthSchema,
+  CrawlerOptions,
+  CrawlResult,
+  Env,
+  ICrawler,
+  ParentSourceDefinition,
+  ScoringConfig,
+  SearchResult,
+  SessionState,
+} from './types.js';

package/src/logger.ts

@@ -0,0 +1,10 @@
+import pino from 'pino';
+
+/**
+ * SDK logger instance
+ * Uses its own pino instance, separate from the main app logger
+ */
+export const sdkLogger = pino({
+  name: 'owletto-sdk',
+  level: process.env.LOG_LEVEL || 'info',
+});