@chaaskit/server 0.1.0

package/dist/tools/web-scrape.js

@@ -0,0 +1,145 @@
+/**
+ * Converts HTML to plain text by stripping tags and normalizing whitespace
+ */
+function htmlToText(html) {
+    return html
+        // Remove script and style elements entirely
+        .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+        .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+        // Remove HTML comments
+        .replace(/<!--[\s\S]*?-->/g, '')
+        // Replace block elements with newlines
+        .replace(/<\/(p|div|h[1-6]|li|tr|br|hr)[^>]*>/gi, '\n')
+        .replace(/<(br|hr)[^>]*\/?>/gi, '\n')
+        // Remove remaining tags
+        .replace(/<[^>]+>/g, '')
+        // Decode common HTML entities
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(parseInt(code, 10)))
+        // Normalize whitespace
+        .replace(/[ \t]+/g, ' ')
+        .replace(/\n\s*\n/g, '\n\n')
+        .trim();
+}
+/**
+ * Truncates text to a maximum length, preserving word boundaries
+ */
+function truncateText(text, maxLength) {
+    if (text.length <= maxLength)
+        return text;
+    const truncated = text.slice(0, maxLength);
+    const lastSpace = truncated.lastIndexOf(' ');
+    if (lastSpace > maxLength * 0.8) {
+        return truncated.slice(0, lastSpace) + '...';
+    }
+    return truncated + '...';
+}
+export const webScrapeTool = {
+    name: 'web-scrape',
+    description: 'Fetches the content of a web page and returns it as plain text. Useful for reading articles, documentation, or any web content.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            url: {
+                type: 'string',
+                description: 'The URL of the web page to fetch',
+            },
+            maxLength: {
+                type: 'number',
+                description: 'Maximum number of characters to return (default: 50000)',
+                default: 50000,
+            },
+        },
+        required: ['url'],
+    },
+    async execute(input, _context) {
+        const url = input.url;
+        const maxLength = input.maxLength || 50000;
+        // Validate URL
+        let parsedUrl;
+        try {
+            parsedUrl = new URL(url);
+        }
+        catch {
+            return {
+                content: [{ type: 'text', text: `Invalid URL: ${url}` }],
+                isError: true,
+            };
+        }
+        // Only allow http/https
+        if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
+            return {
+                content: [{ type: 'text', text: `Invalid protocol: ${parsedUrl.protocol}. Only http and https are allowed.` }],
+                isError: true,
+            };
+        }
+        try {
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), 30000); // 30 second timeout
+            const response = await fetch(url, {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0 (compatible; ChatBot/1.0)',
+                    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                },
+                signal: controller.signal,
+                redirect: 'follow',
+            });
+            clearTimeout(timeout);
+            if (!response.ok) {
+                return {
+                    content: [{ type: 'text', text: `HTTP error: ${response.status} ${response.statusText}` }],
+                    isError: true,
+                };
+            }
+            const contentType = response.headers.get('content-type') || '';
+            // Handle different content types
+            if (contentType.includes('application/json')) {
+                const json = await response.json();
+                const text = JSON.stringify(json, null, 2);
+                return {
+                    content: [{
+                            type: 'text',
+                            text: truncateText(text, maxLength),
+                        }],
+                };
+            }
+            if (contentType.includes('text/plain')) {
+                const text = await response.text();
+                return {
+                    content: [{
+                            type: 'text',
+                            text: truncateText(text, maxLength),
+                        }],
+                };
+            }
+            // Default: treat as HTML
+            const html = await response.text();
+            const text = htmlToText(html);
+            return {
+                content: [{
+                        type: 'text',
+                        text: truncateText(text, maxLength),
+                    }],
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : 'Unknown error';
+            if (message.includes('abort')) {
+                return {
+                    content: [{ type: 'text', text: 'Request timed out after 30 seconds' }],
+                    isError: true,
+                };
+            }
+            return {
+                content: [{ type: 'text', text: `Failed to fetch URL: ${message}` }],
+                isError: true,
+            };
+        }
+    },
+};
+//# sourceMappingURL=web-scrape.js.map

package/dist/tools/web-scrape.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"web-scrape.js","sourceRoot":"","sources":["../../src/tools/web-scrape.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,IAAI;QACT,4CAA4C;SAC3C,OAAO,CAAC,mCAAmC,EAAE,EAAE,CAAC;SAChD,OAAO,CAAC,iCAAiC,EAAE,EAAE,CAAC;QAC/C,uBAAuB;SACtB,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;QAChC,uCAAuC;SACtC,OAAO,CAAC,uCAAuC,EAAE,IAAI,CAAC;SACtD,OAAO,CAAC,qBAAqB,EAAE,IAAI,CAAC;QACrC,wBAAwB;SACvB,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;QACxB,8BAA8B;SAC7B,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;QAC3E,uBAAuB;SACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC;SAC3B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,IAAY,EAAE,SAAiB;IACnD,IAAI,IAAI,CAAC,MAAM,IAAI,SAAS;QAAE,OAAO,IAAI,CAAC;IAE1C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAE7C,IAAI,SAAS,GAAG,SAAS,GAAG,GAAG,EAAE,CAAC;QAChC,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,KAAK,CAAC;IAC/C,CAAC;IAED,OAAO,SAAS,GAAG,KAAK,CAAC;AAC3B,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAe;IACvC,IAAI,EAAE,YAAY;IAElB,WAAW,EAAE,iIAAiI;IAE9I,WAAW,EAAE;QACX,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,GAAG,EAAE;gBACH,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,kCAAkC;aAChD;YACD,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,yDAAyD;gBACtE,OAAO,EAAE,KAAK;aACf;SACF;QACD,QAAQ,EAAE,CAAC,KAAK,CAAC;KAClB;IAED,KAAK,CAAC,OAAO,CAAC,KAA8B,EAAE,QAAqB;QACjE,MAAM,GAAG,GAAG,KAAK,CAAC,GAAa,CAAC;QAChC,MAAM,SAAS,GAAI,KAAK,CAAC,SAAoB,IAAI,KAAK,CAAC;QAEvD,eAAe;QACf,IAAI,SAAc,CAAC;QACnB,IAAI,CAAC;YACH,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,EAAE,EAAE,CAAC;gBACxD,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,wBAAwB;QACxB,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtD,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,qBAAqB,SAAS,CAAC,QAAQ,oCAAoC,EAAE,CAAC;gBAC9G,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,oBAAoB;YAEjF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,OAAO,EAAE;oBACP,YAAY,EAAE,uCAAuC;oBACrD,QAAQ,EAAE,iEAAiE;iBAC5E;gBACD,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC;YAEH,YAAY,CAAC,OAAO,CAAC,CAAC;YAEtB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,OAAO;oBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC;oBAC1F,OAAO,EAAE,IAAI;iBACd,CAAC;YACJ,CAAC;YAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAE/D,iCAAiC;YACjC,IAAI,WAAW,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC3C,OAAO;oBACL,OAAO,EAAE,CAAC;4BACR,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC;yBACpC,CAAC;iBACH,CAAC;YACJ,CAAC;YAED,IAAI,WAAW,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACvC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,OAAO;oBACL,OAAO,EAAE,CAAC;4BACR,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC;yBACpC,CAAC;iBACH,CAAC;YACJ,CAAC;YAED,yBAAyB;YACzB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;YAE9B,OAAO;gBACL,OAAO,EAAE,CAAC;wBACR,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC;qBACpC,CAAC;aACH,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAEzE,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9B,OAAO;oBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,oCAAoC,EAAE,CAAC;oBACvE,OAAO,EAAE,IAAI;iBACd,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,wBAAwB,OAAO,EAAE,EAAE,CAAC;gBACpE,OAAO,EAAE,IAAI;aACd,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC"}

package/package.json

@@ -0,0 +1,93 @@
+{
+  "name": "@chaaskit/server",
+  "version": "0.1.0",
+  "description": "Express.js backend server for ChaasKit AI chat applications",
+  "license": "MIT",
+  "author": "Matt Ferrante <@ferrants>",
+  "homepage": "https://chaaskit.com",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ferrants/chaaskit.git",
+    "directory": "packages/server"
+  },
+  "bugs": {
+    "url": "https://github.com/ferrants/chaaskit/issues"
+  },
+  "keywords": ["chaaskit", "ai", "chat", "saas", "server", "express", "anthropic", "openai"],
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./loaders": {
+      "types": "./dist/loaders/index.d.ts",
+      "import": "./dist/loaders/index.js"
+    }
+  },
+  "bin": {
+    "chaaskit-server": "./dist/bin/cli.js",
+    "chaaskit-worker": "./dist/queue/cli.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx watch --clear-screen=false --watch=../../config --watch=src src/bin/cli.ts",
+    "start": "node dist/bin/cli.js",
+    "typecheck": "tsc --noEmit",
+    "clean": "rm -rf dist"
+  },
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.71.0",
+    "@chaaskit/db": "^0.1.0",
+    "@chaaskit/shared": "^0.1.0",
+    "@modelcontextprotocol/sdk": "^1.0.0",
+    "bcryptjs": "^2.4.3",
+    "cookie-parser": "^1.4.6",
+    "cors": "^2.8.5",
+    "cron-parser": "^5.5.0",
+    "dotenv": "^16.4.1",
+    "express": "^4.18.2",
+    "express-rate-limit": "^7.1.5",
+    "helmet": "^7.1.0",
+    "jiti": "^2.4.0",
+    "jsonwebtoken": "^9.0.2",
+    "multer": "^1.4.5-lts.1",
+    "openai": "^4.28.0",
+    "passport": "^0.7.0",
+    "passport-github2": "^0.1.12",
+    "passport-google-oauth20": "^2.0.0",
+    "passport-local": "^1.0.0",
+    "stripe": "^14.14.0",
+    "uuid": "^9.0.1",
+    "zod": "^3.22.4"
+  },
+  "peerDependencies": {
+    "@aws-sdk/client-sqs": "^3.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@aws-sdk/client-sqs": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@aws-sdk/client-sqs": "^3.0.0",
+    "@types/bcryptjs": "^2.4.6",
+    "@types/cookie-parser": "^1.4.6",
+    "@types/cors": "^2.8.17",
+    "@types/express": "^4.17.21",
+    "@types/jsonwebtoken": "^9.0.5",
+    "@types/multer": "^1.4.11",
+    "@types/passport": "^1.0.16",
+    "@types/passport-github2": "^1.2.9",
+    "@types/passport-google-oauth20": "^2.0.14",
+    "@types/passport-local": "^1.0.38",
+    "@types/uuid": "^9.0.8",
+    "tsx": "^4.7.0",
+    "typescript": "^5.3.3"
+  }
+}