@knowledgesdk/node 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 KnowledgeSDK
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,495 @@
+# @knowledgesdk/node
+
+Official Node.js SDK for the [KnowledgeSDK](https://knowledgesdk.com) API. Extract structured knowledge from any website — business profiles, content, screenshots, sitemaps, and more.
+
+## Installation
+
+```bash
+npm install @knowledgesdk/node
+```
+
+```bash
+yarn add @knowledgesdk/node
+```
+
+## Quick Start
+
+```typescript
+import { KnowledgeSDK } from '@knowledgesdk/node';
+
+const client = new KnowledgeSDK('sk_ks_your_api_key');
+
+// Run the full extraction pipeline on a website
+const result = await client.extract.run('https://example.com');
+console.log(result.business.businessName);
+console.log(result.knowledgeItems);
+```
+
+## Authentication
+
+All API calls require an API key. Keys are prefixed with `sk_ks_`. Pass your key to the constructor:
+
+```typescript
+const client = new KnowledgeSDK('sk_ks_your_api_key');
+```
+
+You can also set `KNOWLEDGE_SDK_API_KEY` as an environment variable and pass it explicitly:
+
+```typescript
+const client = new KnowledgeSDK(process.env.KNOWLEDGE_SDK_API_KEY!);
+```
+
+## Configuration
+
+```typescript
+const client = new KnowledgeSDK('sk_ks_your_api_key', {
+  baseUrl: 'https://api.knowledgesdk.com',  // default
+  maxRetries: 3,                             // default — retries on 429 and 5xx
+  timeout: 30000,                            // default — 30 seconds
+});
+```
+
+## Resources
+
+### `extract`
+
+Run the full pipeline against a URL: scrape, classify, and return structured knowledge items.
+
+#### Synchronous
+
+```typescript
+const result = await client.extract.run('https://acme.com', {
+  maxPages: 20,
+});
+
+console.log(result.business.businessName);       // "Acme Corp"
+console.log(result.business.industrySector);     // "SaaS"
+console.log(result.business.confidenceScore);    // 0.92
+console.log(result.pagesScraped);                // 18
+console.log(result.sitemapUrls);                 // 54
+console.log(result.knowledgeItems.length);       // 12
+
+result.knowledgeItems.forEach((item) => {
+  console.log(`[${item.category}] ${item.title}`);
+  console.log(item.content);
+});
+```
+
+**`ExtractResult` shape:**
+
+```typescript
+{
+  business: {
+    businessName: string;
+    businessType: string;
+    industrySector: string;
+    targetAudience: string;
+    description: string;
+    valueProposition: string;
+    painPoints: string[];
+    uniqueSellingPoints: string[];
+    keyInsights: string[];
+    confidenceScore: number;    // 0-1
+  };
+  knowledgeItems: Array<{
+    title: string;
+    description: string;
+    content: string;
+    category: string;
+    source: string;             // URL of source page
+  }>;
+  pagesScraped: number;
+  sitemapUrls: number;
+}
+```
+
+#### Asynchronous
+
+For long-running extractions, use `runAsync` to get a job ID and poll for the result:
+
+```typescript
+const { jobId, status } = await client.extract.runAsync('https://acme.com', {
+  maxPages: 50,
+  callbackUrl: 'https://your-server.com/webhooks/knowledgesdk',
+});
+
+// Poll until complete
+const job = await client.jobs.poll(jobId, {
+  intervalMs: 3000,   // check every 3 seconds
+  timeoutMs: 300000,  // give up after 5 minutes
+});
+
+if (job.status === 'completed') {
+  const result = job.result as ExtractResult;
+  console.log(result.business.businessName);
+}
+```
+
+---
+
+### `scrape`
+
+Scrape a single page and receive its content as Markdown along with metadata.
+
+```typescript
+const page = await client.scrape.run('https://acme.com/pricing');
+
+console.log(page.title);        // "Pricing — Acme Corp"
+console.log(page.description);  // "Simple, transparent pricing..."
+console.log(page.markdown);     // Full page content in Markdown
+console.log(page.links);        // Array of hrefs found on the page
+```
+
+**`ScrapeResult` shape:**
+
+```typescript
+{
+  url: string;
+  markdown: string;
+  title: string | null;
+  description: string | null;
+  links: string[];
+}
+```
+
+---
+
+### `classify`
+
+Classify a business by analyzing its website. Returns a structured profile without scraping the full site.
+
+```typescript
+const classification = await client.classify.run('https://acme.com');
+
+console.log(classification.businessName);       // "Acme Corp"
+console.log(classification.businessType);       // "B2B Software"
+console.log(classification.industrySector);     // "Project Management"
+console.log(classification.targetAudience);     // "SMBs and mid-market teams"
+console.log(classification.valueProposition);   // "Simplify team collaboration"
+console.log(classification.painPoints);         // ["Too many tools", "Poor visibility"]
+console.log(classification.uniqueSellingPoints); // ["Real-time sync", "One-click reporting"]
+console.log(classification.confidenceScore);    // 0.89
+```
+
+---
+
+### `screenshot`
+
+Capture a full-page screenshot of any URL. Returns a base64-encoded PNG.
+
+```typescript
+const { url, screenshot } = await client.screenshot.run('https://acme.com');
+
+// Write to disk
+import { writeFileSync } from 'fs';
+const buffer = Buffer.from(screenshot, 'base64');
+writeFileSync('screenshot.png', buffer);
+
+// Or use inline in HTML
+const dataUrl = `data:image/png;base64,${screenshot}`;
+```
+
+---
+
+### `sitemap`
+
+Discover all publicly accessible URLs for a website via its sitemap or by crawling.
+
+```typescript
+const { url, urls, count } = await client.sitemap.run('https://acme.com');
+
+console.log(`Found ${count} URLs`);
+urls.forEach((u) => console.log(u));
+```
+
+**`SitemapResult` shape:**
+
+```typescript
+{
+  url: string;
+  urls: string[];
+  count: number;
+}
+```
+
+---
+
+### `search`
+
+Perform a semantic search across your indexed knowledge items.
+
+```typescript
+const results = await client.search.run('how do I cancel my subscription', {
+  limit: 10,
+});
+
+console.log(`${results.total} results for "${results.query}"`);
+
+results.hits.forEach((hit) => {
+  console.log(`[score: ${hit.score.toFixed(2)}] ${hit.title}`);
+  console.log(`Category: ${hit.category} | Source: ${hit.source}`);
+  console.log(hit.content);
+});
+```
+
+**`SearchResult` shape:**
+
+```typescript
+{
+  hits: Array<{
+    id: string;
+    title: string;
+    content: string;
+    category: string;
+    source: string;
+    score: number;       // relevance score, higher is better
+  }>;
+  total: number;
+  query: string;
+}
+```
+
+---
+
+### `webhooks`
+
+Register webhook endpoints to receive real-time event notifications.
+
+#### Create a webhook
+
+```typescript
+const webhook = await client.webhooks.create({
+  url: 'https://your-server.com/webhooks/knowledgesdk',
+  events: ['extract.completed', 'extract.failed'],
+  displayName: 'My Production Webhook',
+});
+
+console.log(webhook.id);     // "wh_abc123"
+console.log(webhook.status); // "active"
+```
+
+#### List webhooks
+
+```typescript
+const webhooks = await client.webhooks.list();
+webhooks.forEach((wh) => {
+  console.log(`${wh.id}: ${wh.url} — ${wh.status}`);
+});
+```
+
+#### Delete a webhook
+
+```typescript
+await client.webhooks.delete('wh_abc123');
+```
+
+#### Test a webhook
+
+Send a test payload to verify your endpoint is reachable:
+
+```typescript
+await client.webhooks.test('wh_abc123');
+```
+
+---
+
+### `jobs`
+
+Retrieve or poll the result of an asynchronous job.
+
+#### Get a job by ID
+
+```typescript
+const job = await client.jobs.get('job_abc123');
+
+console.log(job.status);      // 'pending' | 'processing' | 'completed' | 'failed'
+console.log(job.createdAt);
+console.log(job.completedAt);
+```
+
+#### Poll until complete
+
+```typescript
+import { TimeoutError } from '@knowledgesdk/node';
+
+try {
+  const job = await client.jobs.poll('job_abc123', {
+    intervalMs: 2000,    // poll every 2 seconds (default)
+    timeoutMs: 120000,   // give up after 2 minutes (default)
+  });
+
+  if (job.status === 'completed') {
+    console.log('Job completed:', job.result);
+  } else {
+    console.error('Job failed:', job.error);
+  }
+} catch (err) {
+  if (err instanceof TimeoutError) {
+    console.error('Job timed out — try polling again later');
+  }
+}
+```
+
+---
+
+## Error Handling
+
+All errors extend `KnowledgeSDKError` and carry structured metadata.
+
+```typescript
+import {
+  KnowledgeSDK,
+  KnowledgeSDKError,
+  APIError,
+  AuthenticationError,
+  NetworkError,
+  RateLimitError,
+  TimeoutError,
+} from '@knowledgesdk/node';
+
+const client = new KnowledgeSDK('sk_ks_your_api_key');
+
+try {
+  const result = await client.extract.run('https://acme.com');
+} catch (err) {
+  if (err instanceof AuthenticationError) {
+    console.error('Invalid API key:', err.message);
+  } else if (err instanceof RateLimitError) {
+    console.error('Rate limit hit. Retry after:', err.retryAfter);
+  } else if (err instanceof TimeoutError) {
+    console.error('Request timed out:', err.message);
+  } else if (err instanceof NetworkError) {
+    console.error('Network error:', err.message);
+  } else if (err instanceof APIError) {
+    console.error(`API error ${err.statusCode}:`, err.message);
+    console.error('Error code:', err.code);
+    console.error('Request ID:', err.requestId);
+  } else if (err instanceof KnowledgeSDKError) {
+    console.error('KnowledgeSDK error:', err.message);
+  } else {
+    throw err;
+  }
+}
+```
+
+### Error Classes
+
+| Class | Description |
+|---|---|
+| `KnowledgeSDKError` | Base class for all SDK errors |
+| `APIError` | 4xx/5xx responses from the API |
+| `AuthenticationError` | Missing or invalid API key (401) |
+| `NetworkError` | Network connectivity issues |
+| `RateLimitError` | API rate limit exceeded (429) |
+| `TimeoutError` | Request or job polling timed out |
+
+All errors expose:
+- `message` — human-readable description
+- `statusCode` — HTTP status code (where applicable)
+- `code` — machine-readable error code
+- `requestId` — request ID from the API (for support)
+- `data` — raw response body (where available)
+
+---
+
+## Debug Mode
+
+Enable request/response logging for development:
+
+```typescript
+const client = new KnowledgeSDK('sk_ks_your_api_key');
+client.setDebugMode(true);
+
+// All requests and responses will now be printed to the console
+const result = await client.scrape.run('https://acme.com');
+```
+
+---
+
+## Advanced Usage
+
+### Custom headers
+
+```typescript
+client.setHeaders({
+  'x-custom-header': 'my-value',
+  'x-trace-id': requestId,
+});
+```
+
+### Retry configuration
+
+By default the SDK retries up to 3 times on rate-limit (429) and server errors (5xx) using exponential backoff with jitter. Configure at construction:
+
+```typescript
+const client = new KnowledgeSDK('sk_ks_your_api_key', {
+  maxRetries: 5,
+  timeout: 60000, // 60 seconds
+});
+```
+
+### Full async extraction workflow
+
+```typescript
+import { KnowledgeSDK, ExtractResult, TimeoutError } from '@knowledgesdk/node';
+
+const client = new KnowledgeSDK(process.env.KNOWLEDGE_SDK_API_KEY!);
+
+async function extractWebsite(url: string): Promise<ExtractResult> {
+  // Kick off async job
+  const { jobId } = await client.extract.runAsync(url, {
+    maxPages: 100,
+    callbackUrl: 'https://your-server.com/webhooks/knowledgesdk',
+  });
+
+  console.log(`Started job ${jobId}, polling for result...`);
+
+  // Poll until complete (up to 10 minutes)
+  const job = await client.jobs.poll(jobId, {
+    intervalMs: 5000,
+    timeoutMs: 600000,
+  });
+
+  if (job.status === 'failed') {
+    throw new Error(`Extraction failed: ${job.error}`);
+  }
+
+  return job.result as ExtractResult;
+}
+
+const result = await extractWebsite('https://large-site.com');
+console.log(`Extracted ${result.knowledgeItems.length} knowledge items`);
+```
+
+---
+
+## TypeScript
+
+The SDK is written in TypeScript and exports all types. No additional `@types` package is required.
+
+```typescript
+import type {
+  ExtractResult,
+  ExtractOptions,
+  ExtractAsyncResult,
+  KnowledgeItem,
+  BusinessProfile,
+  ScrapeResult,
+  BusinessClassification,
+  ScreenshotResult,
+  SitemapResult,
+  SearchResult,
+  SearchHit,
+  WebhookFull,
+  WebhookCreateOptions,
+  JobResult,
+  JobStatus,
+  KnowledgeSDKOptions,
+} from '@knowledgesdk/node';
+```
+
+---
+
+## License
+
+MIT

package/dist/api/classify.d.ts

@@ -0,0 +1,24 @@
+import { HttpClient } from '../utils/http-client';
+export interface BusinessClassification {
+    businessName: string;
+    businessType: string;
+    industrySector: string;
+    targetAudience: string;
+    description: string;
+    valueProposition: string;
+    painPoints: string[];
+    uniqueSellingPoints: string[];
+    keyInsights: string[];
+    confidenceScore: number;
+}
+export declare class Classify {
+    private httpClient;
+    constructor(httpClient: HttpClient);
+    /**
+     * Classify a business by analyzing its website URL.
+     * Returns a structured business profile with industry, target audience, and key insights.
+     * @param url The URL of the business website to classify
+     * @returns A structured business classification
+     */
+    run(url: string): Promise<BusinessClassification>;
+}

package/dist/api/classify.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Classify = void 0;
+class Classify {
+    constructor(httpClient) {
+        this.httpClient = httpClient;
+    }
+    /**
+     * Classify a business by analyzing its website URL.
+     * Returns a structured business profile with industry, target audience, and key insights.
+     * @param url The URL of the business website to classify
+     * @returns A structured business classification
+     */
+    async run(url) {
+        return this.httpClient.post('/classify', { url });
+    }
+}
+exports.Classify = Classify;
+//# sourceMappingURL=classify.js.map

package/dist/api/classify.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"classify.js","sourceRoot":"","sources":["../../src/api/classify.ts"],"names":[],"mappings":";;;AAeA,MAAa,QAAQ;IAGnB,YAAY,UAAsB;QAChC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,GAAG,CAAC,GAAW;QACnB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAyB,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5E,CAAC;CACF;AAhBD,4BAgBC"}

package/dist/api/extract.d.ts

@@ -0,0 +1,118 @@
+import { HttpClient } from '../utils/http-client';
+export interface KnowledgeItem {
+    title: string;
+    description: string;
+    content: string;
+    category: string;
+    source: string;
+}
+export interface BusinessProfile {
+    businessName: string;
+    businessType: string;
+    industrySector: string;
+    targetAudience: string;
+    description: string;
+    valueProposition: string;
+    painPoints: string[];
+    uniqueSellingPoints: string[];
+    keyInsights: string[];
+    confidenceScore: number;
+}
+export interface ExtractResult {
+    business: BusinessProfile;
+    knowledgeItems: KnowledgeItem[];
+    pagesScraped: number;
+    sitemapUrls: number;
+}
+export interface ExtractOptions {
+    maxPages?: number;
+}
+export interface ExtractStreamOptions {
+    maxPages?: number;
+}
+export type ExtractStreamEvent = {
+    type: 'connected';
+    message: string;
+} | {
+    type: 'progress';
+    message: string;
+} | {
+    type: 'business_classified';
+    business: {
+        businessName: string;
+        businessType: string;
+        industry: string;
+        description: string;
+    };
+} | {
+    type: 'pages_planned';
+    pages: Array<{
+        url: string;
+        purpose: string;
+    }>;
+} | {
+    type: 'page_scraped';
+    url: string;
+    index: number;
+    total: number;
+    status: 'done' | 'failed';
+} | {
+    type: 'urls_triaged';
+    suggestedUrls: Array<{
+        url: string;
+        reason: string;
+    }>;
+} | {
+    type: 'complete';
+    result: ExtractResult;
+} | {
+    type: 'error';
+    message: string;
+};
+export interface ExtractAsyncOptions {
+    maxPages?: number;
+    callbackUrl?: string;
+}
+export interface ExtractAsyncResult {
+    jobId: string;
+    status: string;
+}
+export declare class Extract {
+    private httpClient;
+    constructor(httpClient: HttpClient);
+    /**
+     * Run a synchronous extraction pipeline against a URL.
+     * Scrapes the site, classifies the business, and returns structured knowledge items.
+     * @param url The URL to extract knowledge from
+     * @param options Optional extraction options
+     * @returns The full extraction result including business profile and knowledge items
+     */
+    run(url: string, options?: ExtractOptions): Promise<ExtractResult>;
+    /**
+     * Start an asynchronous extraction pipeline and return a job ID.
+     * Use jobs.poll() or jobs.get() to retrieve the result when complete.
+     * @param url The URL to extract knowledge from
+     * @param options Optional async extraction options including a callbackUrl
+     * @returns The job ID and initial status
+     */
+    runAsync(url: string, options?: ExtractAsyncOptions): Promise<ExtractAsyncResult>;
+    /**
+     * Stream extraction progress as server-sent events.
+     * Yields typed events as the pipeline runs: classification, page discovery,
+     * per-page scraping, and the final complete result.
+     * Requires Node.js 18+ (native fetch).
+     *
+     * @example
+     * ```typescript
+     * for await (const event of client.extract.runStream('https://stripe.com')) {
+     *   if (event.type === 'page_scraped') {
+     *     console.log(`Scraped ${event.index + 1}/${event.total}: ${event.url}`);
+     *   }
+     *   if (event.type === 'complete') {
+     *     console.log(event.result.knowledgeItems);
+     *   }
+     * }
+     * ```
+     */
+    runStream(url: string, options?: ExtractStreamOptions): AsyncGenerator<ExtractStreamEvent>;
+}