@crawlith/core 0.1.1 → 0.1.2

package/dist/utils/secureConfig.d.ts

@@ -0,0 +1,23 @@
+export interface CrawlithConfig {
+    [section: string]: {
+        key?: string;
+        createdAt?: number;
+        [key: string]: unknown;
+    };
+}
+/**
+ * Resolve the canonical Crawlith config file path.
+ */
+export declare function getCrawlithConfigPath(): string;
+/**
+ * Return section config, or undefined if config file/section does not exist.
+ */
+export declare function getConfigSection(section: string): CrawlithConfig[string] | undefined;
+/**
+ * Encrypt and persist a section API key in ~/.crawlith/config.json.
+ */
+export declare function setEncryptedConfigKey(section: string, apiKey: string): void;
+/**
+ * Get and decrypt the API key for a config section.
+ */
+export declare function getDecryptedConfigKey(section: string): string;

package/dist/utils/secureConfig.js

@@ -0,0 +1,128 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import crypto from 'node:crypto';
+const CONFIG_DIR = path.join(os.homedir(), '.crawlith');
+const CONFIG_PATH = path.join(CONFIG_DIR, 'config.json');
+/**
+ * Resolve the canonical Crawlith config file path.
+ */
+export function getCrawlithConfigPath() {
+    return CONFIG_PATH;
+}
+/**
+ * Return section config, or undefined if config file/section does not exist.
+ */
+export function getConfigSection(section) {
+    const config = readConfigFile(false);
+    if (!config)
+        return undefined;
+    return config[section];
+}
+/**
+ * Encrypt and persist a section API key in ~/.crawlith/config.json.
+ */
+export function setEncryptedConfigKey(section, apiKey) {
+    const config = readConfigFile(false) || {};
+    config[section] = {
+        ...(config[section] || {}),
+        key: encryptString(apiKey),
+        createdAt: Math.floor(Date.now() / 1000)
+    };
+    writeConfigFile(config);
+}
+/**
+ * Get and decrypt the API key for a config section.
+ */
+export function getDecryptedConfigKey(section) {
+    if (!fs.existsSync(CONFIG_PATH)) {
+        throw new Error(`Missing ${section} config. Run: crawlith config ${section} set <api_key>`);
+    }
+    const config = readConfigFile(true);
+    if (!config) {
+        throw new Error(`Missing ${section} config. Run: crawlith config ${section} set <api_key>`);
+    }
+    const payload = config[section]?.key;
+    if (!payload || typeof payload !== 'string') {
+        throw new Error(`Missing ${section} key in config. Run: crawlith config ${section} set <api_key>`);
+    }
+    return decryptString(payload);
+}
+/**
+ * Read config file from disk.
+ */
+function readConfigFile(required) {
+    if (!fs.existsSync(CONFIG_PATH)) {
+        if (required) {
+            throw new Error('Missing config file. Run: crawlith config <service> set <api_key>');
+        }
+        return null;
+    }
+    try {
+        return JSON.parse(fs.readFileSync(CONFIG_PATH, 'utf8'));
+    }
+    catch {
+        throw new Error('Corrupt config file at ~/.crawlith/config.json. Refusing to continue.');
+    }
+}
+/**
+ * Persist config to disk with secure permissions.
+ */
+function writeConfigFile(config) {
+    if (!fs.existsSync(CONFIG_DIR)) {
+        fs.mkdirSync(CONFIG_DIR, { recursive: true, mode: 0o700 });
+    }
+    fs.writeFileSync(CONFIG_PATH, JSON.stringify(config, null, 2), { encoding: 'utf8', mode: 0o600 });
+    fs.chmodSync(CONFIG_PATH, 0o600);
+}
+/**
+ * Build a machine-bound secret so encrypted config blobs are not portable across systems.
+ */
+function getMachineSecret() {
+    return `${os.hostname()}::${os.userInfo().username}`;
+}
+/**
+ * Encrypt plaintext using AES-256-GCM and scrypt-derived key.
+ */
+function encryptString(plaintext) {
+    const salt = crypto.randomBytes(16);
+    const iv = crypto.randomBytes(12);
+    const key = crypto.scryptSync(getMachineSecret(), salt, 32);
+    const cipher = crypto.createCipheriv('aes-256-gcm', key, iv);
+    const encrypted = Buffer.concat([cipher.update(plaintext, 'utf8'), cipher.final()]);
+    const payload = {
+        salt: salt.toString('base64'),
+        iv: iv.toString('base64'),
+        tag: cipher.getAuthTag().toString('base64'),
+        data: encrypted.toString('base64')
+    };
+    return Buffer.from(JSON.stringify(payload), 'utf8').toString('base64');
+}
+/**
+ * Decrypt an encrypted base64 payload from config.json.
+ */
+function decryptString(encodedPayload) {
+    let payload;
+    try {
+        payload = JSON.parse(Buffer.from(encodedPayload, 'base64').toString('utf8'));
+    }
+    catch {
+        throw new Error('Corrupt config payload: unable to parse encrypted key data.');
+    }
+    if (!payload?.salt || !payload?.iv || !payload?.tag || !payload?.data) {
+        throw new Error('Corrupt config payload: required encryption fields are missing.');
+    }
+    try {
+        const salt = Buffer.from(payload.salt, 'base64');
+        const iv = Buffer.from(payload.iv, 'base64');
+        const tag = Buffer.from(payload.tag, 'base64');
+        const data = Buffer.from(payload.data, 'base64');
+        const key = crypto.scryptSync(getMachineSecret(), salt, 32);
+        const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv);
+        decipher.setAuthTag(tag);
+        return Buffer.concat([decipher.update(data), decipher.final()]).toString('utf8');
+    }
+    catch {
+        throw new Error('Unable to decrypt config key. Config may be invalid or tied to another machine/user.');
+    }
+}

package/package.json

@@ -1,6 +1,8 @@
 {
   "name": "@crawlith/core",
-  "version": "0.1.1",
+  "license": "Apache-2.0",
+  "version": "0.1.2",
+  "description": "Headless intelligence engine for Crawlith. Handles crawling, graph analysis, scoring, and data persistence.",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -11,19 +13,23 @@
       "default": "./dist/index.js"
     }
   },
+  "files": [
+    "dist"
+  ],
   "dependencies": {
     "better-sqlite3": "^12.6.2",
-    "chalk": "^5.3.0",
     "cheerio": "^1.0.0-rc.12",
+    "commander": "^12.1.0",
     "p-limit": "^7.3.0",
     "robots-parser": "^3.0.1",
-    "undici": "^6.13.0",
-    "vite": "7.3.1"
+    "undici": "^6.13.0"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.13",
+    "@types/cheerio": "0.22.35",
     "@types/node": "^20.12.7",
     "typescript": "^5.4.5",
+    "vite": "7.3.1",
     "vitest": "^4.0.18"
   },
   "scripts": {

package/CHANGELOG.md

@@ -1,13 +0,0 @@
-# @crawlith/core
-
-## 0.1.1
-
-### Patch Changes
-
-- Bump all packages to next patch version.
-
-## 0.1.0
-
-### Minor Changes
-
-- First test

package/dist/db/schema.d.ts

@@ -1,2 +0,0 @@
-import { Database } from 'better-sqlite3';
-export declare function initSchema(db: Database): void;

package/dist/graph/cluster.d.ts

@@ -1,6 +0,0 @@
-import { Graph, ClusterInfo } from './graph.js';
-/**
- * Detects content clusters using 64-bit SimHash and Hamming Distance.
- * Uses band optimization to reduce O(n^2) comparisons.
- */
-export declare function detectContentClusters(graph: Graph, threshold?: number, minSize?: number): ClusterInfo[];

package/dist/graph/cluster.js

@@ -1,221 +0,0 @@
-import { SimHash } from './simhash.js';
-import { load } from 'cheerio';
-/**
- * Detects content clusters using 64-bit SimHash and Hamming Distance.
- * Uses band optimization to reduce O(n^2) comparisons.
- */
-export function detectContentClusters(graph, threshold = 10, minSize = 3) {
-    const nodes = graph.getNodes().filter(n => n.simhash && n.status === 200);
-    if (nodes.length === 0)
-        return [];
-    const adjacency = new Map();
-    // Banding Optimization (4 bands of 16 bits)
-    // Note: For threshold > 3, this is a heuristic and may miss some pairs,
-    // but it dramatically reduces the search space as requested.
-    const buckets = Array.from({ length: SimHash.BANDS }, () => new Map());
-    for (const node of nodes) {
-        const hash = BigInt(node.simhash);
-        const bandValues = SimHash.getBands(hash);
-        bandValues.forEach((bandValue, b) => {
-            if (!buckets[b].has(bandValue)) {
-                buckets[b].set(bandValue, new Set());
-            }
-            buckets[b].get(bandValue).add(node.url);
-        });
-    }
-    const checkedPairs = new Set();
-    for (let b = 0; b < SimHash.BANDS; b++) {
-        for (const bucket of buckets[b].values()) {
-            if (bucket.size < 2)
-                continue;
-            const bucketNodes = Array.from(bucket);
-            for (let i = 0; i < bucketNodes.length; i++) {
-                for (let j = i + 1; j < bucketNodes.length; j++) {
-                    const u1 = bucketNodes[i];
-                    const u2 = bucketNodes[j];
-                    if (u1 === u2)
-                        continue;
-                    const pairKey = u1 < u2 ? `${u1}|${u2}` : `${u2}|${u1}`;
-                    if (checkedPairs.has(pairKey))
-                        continue;
-                    checkedPairs.add(pairKey);
-                    const n1 = graph.nodes.get(u1);
-                    const n2 = graph.nodes.get(u2);
-                    const dist = SimHash.hammingDistance(BigInt(n1.simhash), BigInt(n2.simhash));
-                    if (dist <= threshold) {
-                        if (!adjacency.has(u1))
-                            adjacency.set(u1, new Set());
-                        if (!adjacency.has(u2))
-                            adjacency.set(u2, new Set());
-                        adjacency.get(u1).add(u2);
-                        adjacency.get(u2).add(u1);
-                    }
-                }
-            }
-        }
-    }
-    // Find connected components (Clusters)
-    const visited = new Set();
-    const clusters = [];
-    for (const node of nodes) {
-        if (visited.has(node.url))
-            continue;
-        const component = [];
-        const queue = [node.url];
-        visited.add(node.url);
-        while (queue.length > 0) {
-            const current = queue.shift();
-            component.push(current);
-            const neighbors = adjacency.get(current);
-            if (neighbors) {
-                for (const neighbor of neighbors) {
-                    if (!visited.has(neighbor)) {
-                        visited.add(neighbor);
-                        queue.push(neighbor);
-                    }
-                }
-            }
-        }
-        if (component.length >= minSize) {
-            clusters.push(component);
-        }
-    }
-    // Sort clusters by size (descending) then by primary URL (ascending) for deterministic IDs
-    clusters.sort((a, b) => {
-        if (b.length !== a.length)
-            return b.length - a.length;
-        const aPrimary = selectPrimaryUrl(a, graph);
-        const bPrimary = selectPrimaryUrl(b, graph);
-        return aPrimary.localeCompare(bPrimary);
-    });
-    const clusterInfos = [];
-    clusters.forEach((memberUrls, index) => {
-        const clusterId = index + 1;
-        const clusterNodes = memberUrls.map(url => graph.nodes.get(url));
-        for (const node of clusterNodes) {
-            node.clusterId = clusterId;
-        }
-        const primaryUrl = selectPrimaryUrl(memberUrls, graph);
-        const risk = calculateClusterRisk(clusterNodes);
-        const sharedPathPrefix = findSharedPathPrefix(memberUrls);
-        clusterInfos.push({
-            id: clusterId,
-            count: memberUrls.length,
-            primaryUrl,
-            risk,
-            sharedPathPrefix
-        });
-    });
-    graph.contentClusters = clusterInfos;
-    return clusterInfos;
-}
-/**
- * Selects the primary URL for a cluster based on:
- * 1. Highest PageRank
- * 2. Shortest URL
- * 3. Lexicographic fallback
- */
-function selectPrimaryUrl(urls, graph) {
-    return urls.reduce((best, current) => {
-        const nBest = graph.nodes.get(best);
-        const nCurrent = graph.nodes.get(current);
-        if ((nCurrent.pageRank || 0) > (nBest.pageRank || 0))
-            return current;
-        if ((nCurrent.pageRank || 0) < (nBest.pageRank || 0))
-            return best;
-        if (current.length < best.length)
-            return current;
-        if (current.length > best.length)
-            return best;
-        return current.localeCompare(best) < 0 ? current : best;
-    });
-}
-/**
- * Calculates cannibalization risk based on title and H1 similarity within the cluster.
- */
-function calculateClusterRisk(nodes) {
-    if (nodes.length <= 1)
-        return 'low';
-    // Count title and H1 occurrences
-    const titleCounts = new Map();
-    const h1Counts = new Map();
-    let processedCount = 0;
-    for (const node of nodes) {
-        if (!node.html)
-            continue;
-        try {
-            const $ = load(node.html);
-            const title = $('title').text().trim().toLowerCase();
-            const h1 = $('h1').first().text().trim().toLowerCase();
-            if (title) {
-                titleCounts.set(title, (titleCounts.get(title) || 0) + 1);
-            }
-            if (h1) {
-                h1Counts.set(h1, (h1Counts.get(h1) || 0) + 1);
-            }
-            processedCount++;
-        }
-        catch {
-            // Ignore parsing errors
-        }
-    }
-    // If we couldn't parse enough content (e.g., no HTML stored), fallback to size-based heuristic
-    if (processedCount < nodes.length * 0.5) {
-        if (nodes.length > 5)
-            return 'high';
-        if (nodes.length > 2)
-            return 'medium';
-        return 'low';
-    }
-    // Calculate duplicate ratios
-    let duplicateTitleCount = 0;
-    let duplicateH1Count = 0;
-    for (const count of titleCounts.values()) {
-        if (count > 1)
-            duplicateTitleCount += count;
-    }
-    for (const count of h1Counts.values()) {
-        if (count > 1)
-            duplicateH1Count += count;
-    }
-    const titleDupeRatio = duplicateTitleCount / nodes.length;
-    const h1DupeRatio = duplicateH1Count / nodes.length;
-    // Heuristic 1: High Risk
-    // Significant overlap in Titles OR H1s (e.g., > 30% of cluster members are duplicates)
-    if (titleDupeRatio > 0.3 || h1DupeRatio > 0.3) {
-        return 'high';
-    }
-    // Heuristic 2: Medium Risk
-    // Any overlap, or very large clusters (potential template issues or thin content)
-    if (titleDupeRatio > 0 || h1DupeRatio > 0 || nodes.length > 10) {
-        return 'medium';
-    }
-    // Heuristic 3: Low Risk
-    // Unique content and manageable cluster size
-    return 'low';
-}
-/**
- * Finds the common path prefix among a set of URLs.
- */
-function findSharedPathPrefix(urls) {
-    if (urls.length < 2)
-        return undefined;
-    try {
-        const paths = urls.map(u => new URL(u).pathname.split('/').filter(Boolean));
-        const first = paths[0];
-        const common = [];
-        for (let i = 0; i < first.length; i++) {
-            const segment = first[i];
-            if (paths.every(p => p[i] === segment)) {
-                common.push(segment);
-            }
-            else {
-                break;
-            }
-        }
-        return common.length > 0 ? '/' + common.join('/') : undefined;
-    }
-    catch {
-        return undefined;
-    }
-}

package/dist/graph/duplicate.d.ts

@@ -1,10 +0,0 @@
-import { Graph } from './graph.js';
-export interface DuplicateOptions {
-    collapse?: boolean;
-    simhashThreshold?: number;
-}
-/**
- * Detects exact and near duplicates, identifies canonical conflicts,
- * and performs non-destructive collapse of edges.
- */
-export declare function detectDuplicates(graph: Graph, options?: DuplicateOptions): void;