mcpmake 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +691 -0
- package/bin/mcpmake.mjs +2 -0
- package/dist/analyzer/auth-detector.d.ts +12 -0
- package/dist/analyzer/auth-detector.js +142 -0
- package/dist/analyzer/dom-parser.d.ts +10 -0
- package/dist/analyzer/dom-parser.js +259 -0
- package/dist/analyzer/goal-crawler.d.ts +25 -0
- package/dist/analyzer/goal-crawler.js +177 -0
- package/dist/analyzer/hybrid-detector.d.ts +28 -0
- package/dist/analyzer/hybrid-detector.js +96 -0
- package/dist/analyzer/index.d.ts +12 -0
- package/dist/analyzer/index.js +8 -0
- package/dist/analyzer/screenshot-capture.d.ts +29 -0
- package/dist/analyzer/screenshot-capture.js +42 -0
- package/dist/analyzer/selector-builder.d.ts +19 -0
- package/dist/analyzer/selector-builder.js +199 -0
- package/dist/analyzer/semantic-analyzer.d.ts +13 -0
- package/dist/analyzer/semantic-analyzer.js +145 -0
- package/dist/analyzer/site-crawler.d.ts +38 -0
- package/dist/analyzer/site-crawler.js +235 -0
- package/dist/cloud/billing/billing-engine.d.ts +44 -0
- package/dist/cloud/billing/billing-engine.js +81 -0
- package/dist/cloud/billing/credit-store.d.ts +64 -0
- package/dist/cloud/billing/credit-store.js +168 -0
- package/dist/cloud/billing/index.d.ts +4 -0
- package/dist/cloud/billing/index.js +2 -0
- package/dist/cloud/billing/usage-store.d.ts +42 -0
- package/dist/cloud/billing/usage-store.js +85 -0
- package/dist/cloud/billing/usage-tracker.d.ts +38 -0
- package/dist/cloud/billing/usage-tracker.js +95 -0
- package/dist/cloud/build-pipeline.d.ts +39 -0
- package/dist/cloud/build-pipeline.js +310 -0
- package/dist/cloud/build-queue.d.ts +30 -0
- package/dist/cloud/build-queue.js +70 -0
- package/dist/cloud/caddy-manager.d.ts +18 -0
- package/dist/cloud/caddy-manager.js +97 -0
- package/dist/cloud/container-backend.d.ts +62 -0
- package/dist/cloud/container-backend.js +59 -0
- package/dist/cloud/container-manager.d.ts +64 -0
- package/dist/cloud/container-manager.js +301 -0
- package/dist/cloud/crypto.d.ts +27 -0
- package/dist/cloud/crypto.js +63 -0
- package/dist/cloud/db/index.d.ts +27 -0
- package/dist/cloud/db/index.js +53 -0
- package/dist/cloud/db/migrations.d.ts +12 -0
- package/dist/cloud/db/migrations.js +329 -0
- package/dist/cloud/db/pg-store.d.ts +45 -0
- package/dist/cloud/db/pg-store.js +336 -0
- package/dist/cloud/failure-tracker.d.ts +51 -0
- package/dist/cloud/failure-tracker.js +102 -0
- package/dist/cloud/idle-monitor.d.ts +30 -0
- package/dist/cloud/idle-monitor.js +70 -0
- package/dist/cloud/mailer.d.ts +21 -0
- package/dist/cloud/mailer.js +193 -0
- package/dist/cloud/mcp-proxy.d.ts +58 -0
- package/dist/cloud/mcp-proxy.js +203 -0
- package/dist/cloud/metric-samples.d.ts +43 -0
- package/dist/cloud/metric-samples.js +85 -0
- package/dist/cloud/metrics.d.ts +26 -0
- package/dist/cloud/metrics.js +59 -0
- package/dist/cloud/multipart.d.ts +26 -0
- package/dist/cloud/multipart.js +132 -0
- package/dist/cloud/observability.d.ts +27 -0
- package/dist/cloud/observability.js +98 -0
- package/dist/cloud/rate-limiter.d.ts +31 -0
- package/dist/cloud/rate-limiter.js +58 -0
- package/dist/cloud/request-security.d.ts +5 -0
- package/dist/cloud/request-security.js +74 -0
- package/dist/cloud/resource-monitor.d.ts +69 -0
- package/dist/cloud/resource-monitor.js +130 -0
- package/dist/cloud/secret-store.d.ts +38 -0
- package/dist/cloud/secret-store.js +103 -0
- package/dist/cloud/security.d.ts +26 -0
- package/dist/cloud/security.js +142 -0
- package/dist/cloud/server.d.ts +21 -0
- package/dist/cloud/server.js +1079 -0
- package/dist/cloud/shared-state.d.ts +72 -0
- package/dist/cloud/shared-state.js +159 -0
- package/dist/cloud/ssrf.d.ts +43 -0
- package/dist/cloud/ssrf.js +150 -0
- package/dist/cloud/store.d.ts +41 -0
- package/dist/cloud/store.js +75 -0
- package/dist/cloud/stripe.d.ts +78 -0
- package/dist/cloud/stripe.js +317 -0
- package/dist/cloud/telemetry-store.d.ts +53 -0
- package/dist/cloud/telemetry-store.js +108 -0
- package/dist/cloud/web/auth.d.ts +225 -0
- package/dist/cloud/web/auth.js +555 -0
- package/dist/cloud/web/charts.d.ts +70 -0
- package/dist/cloud/web/charts.js +178 -0
- package/dist/cloud/web/csrf.d.ts +14 -0
- package/dist/cloud/web/csrf.js +22 -0
- package/dist/cloud/web/docs.d.ts +40 -0
- package/dist/cloud/web/docs.js +174 -0
- package/dist/cloud/web/router.d.ts +25 -0
- package/dist/cloud/web/router.js +1921 -0
- package/dist/cloud/web/static/alpine.min.js +5 -0
- package/dist/cloud/web/static/favicon.svg +4 -0
- package/dist/cloud/web/static/htmx-sse.js +290 -0
- package/dist/cloud/web/static/htmx.min.js +1 -0
- package/dist/cloud/web/static/style.css +2683 -0
- package/dist/cloud/web/static-server.d.ts +13 -0
- package/dist/cloud/web/static-server.js +73 -0
- package/dist/cloud/web/template-engine.d.ts +27 -0
- package/dist/cloud/web/template-engine.js +146 -0
- package/dist/cloud/web/templates/layouts/admin.hbs +57 -0
- package/dist/cloud/web/templates/layouts/auth.hbs +138 -0
- package/dist/cloud/web/templates/layouts/base.hbs +16 -0
- package/dist/cloud/web/templates/layouts/dashboard.hbs +39 -0
- package/dist/cloud/web/templates/layouts/landing.hbs +82 -0
- package/dist/cloud/web/templates/pages/admin/overview.hbs +123 -0
- package/dist/cloud/web/templates/pages/admin/servers.hbs +129 -0
- package/dist/cloud/web/templates/pages/admin/telemetry.hbs +39 -0
- package/dist/cloud/web/templates/pages/admin/user-edit.hbs +91 -0
- package/dist/cloud/web/templates/pages/admin/users.hbs +179 -0
- package/dist/cloud/web/templates/pages/auth/forgot-password.hbs +25 -0
- package/dist/cloud/web/templates/pages/auth/login.hbs +33 -0
- package/dist/cloud/web/templates/pages/auth/register.hbs +32 -0
- package/dist/cloud/web/templates/pages/auth/reset-password.hbs +34 -0
- package/dist/cloud/web/templates/pages/dashboard/billing.hbs +140 -0
- package/dist/cloud/web/templates/pages/dashboard/create.hbs +173 -0
- package/dist/cloud/web/templates/pages/dashboard/index.hbs +8 -0
- package/dist/cloud/web/templates/pages/dashboard/server-detail.hbs +280 -0
- package/dist/cloud/web/templates/pages/dashboard/server-logs.hbs +35 -0
- package/dist/cloud/web/templates/pages/dashboard/server-metrics.hbs +63 -0
- package/dist/cloud/web/templates/pages/dashboard/servers-partial.hbs +21 -0
- package/dist/cloud/web/templates/pages/dashboard/servers.hbs +44 -0
- package/dist/cloud/web/templates/pages/docs/show.hbs +16 -0
- package/dist/cloud/web/templates/pages/errors/404.hbs +9 -0
- package/dist/cloud/web/templates/pages/errors/500.hbs +8 -0
- package/dist/cloud/web/templates/pages/landing/index.hbs +223 -0
- package/dist/cloud/web/templates/pages/legal/privacy.hbs +71 -0
- package/dist/cloud/web/templates/pages/legal/terms.hbs +73 -0
- package/dist/cloud/web/templates/partials/admin-stats.hbs +52 -0
- package/dist/cloud/web/templates/partials/flash-message.hbs +6 -0
- package/dist/cloud/web/templates/partials/pricing-table.hbs +103 -0
- package/dist/cloud/web/templates/partials/server-card.hbs +19 -0
- package/dist/cloud/web/templates/partials/status-badge.hbs +1 -0
- package/dist/commands/bundle.d.ts +18 -0
- package/dist/commands/bundle.js +82 -0
- package/dist/commands/ci.d.ts +25 -0
- package/dist/commands/ci.js +149 -0
- package/dist/commands/deploy.d.ts +24 -0
- package/dist/commands/deploy.js +145 -0
- package/dist/commands/diff.d.ts +18 -0
- package/dist/commands/diff.js +185 -0
- package/dist/commands/from/describe.d.ts +65 -0
- package/dist/commands/from/describe.js +173 -0
- package/dist/commands/from/har.d.ts +81 -0
- package/dist/commands/from/har.js +255 -0
- package/dist/commands/from/openapi.d.ts +105 -0
- package/dist/commands/from/openapi.js +302 -0
- package/dist/commands/from/postman.d.ts +51 -0
- package/dist/commands/from/postman.js +146 -0
- package/dist/commands/from/target-support.d.ts +11 -0
- package/dist/commands/from/target-support.js +33 -0
- package/dist/commands/from/url.d.ts +75 -0
- package/dist/commands/from/url.js +244 -0
- package/dist/commands/from/website.d.ts +75 -0
- package/dist/commands/from/website.js +284 -0
- package/dist/commands/lint.d.ts +24 -0
- package/dist/commands/lint.js +184 -0
- package/dist/commands/merge.d.ts +18 -0
- package/dist/commands/merge.js +161 -0
- package/dist/commands/publish.d.ts +27 -0
- package/dist/commands/publish.js +334 -0
- package/dist/commands/rescan.d.ts +40 -0
- package/dist/commands/rescan.js +255 -0
- package/dist/commands/update.d.ts +14 -0
- package/dist/commands/update.js +87 -0
- package/dist/commands/verify.d.ts +14 -0
- package/dist/commands/verify.js +71 -0
- package/dist/config/configurable-command.d.ts +13 -0
- package/dist/config/configurable-command.js +70 -0
- package/dist/config/mcpmake-config.d.ts +68 -0
- package/dist/config/mcpmake-config.js +207 -0
- package/dist/docs/cli.md +400 -0
- package/dist/docs/mcp-2026-07-28-migration.md +78 -0
- package/dist/docs/migrate-from-stainless.md +94 -0
- package/dist/docs/quickstart.md +166 -0
- package/dist/docs/show-hn.md +26 -0
- package/dist/docs/website-servers.md +169 -0
- package/dist/emitter/code-writer.d.ts +8 -0
- package/dist/emitter/code-writer.js +25 -0
- package/dist/emitter/index.d.ts +32 -0
- package/dist/emitter/index.js +280 -0
- package/dist/emitter/mcpb-bundler.d.ts +31 -0
- package/dist/emitter/mcpb-bundler.js +172 -0
- package/dist/emitter/project-scaffolder.d.ts +4 -0
- package/dist/emitter/project-scaffolder.js +89 -0
- package/dist/emitter/python-template-loader.d.ts +4 -0
- package/dist/emitter/python-template-loader.js +30 -0
- package/dist/emitter/python-templates/dockerfile.hbs +14 -0
- package/dist/emitter/python-templates/env.example.hbs +6 -0
- package/dist/emitter/python-templates/requirements.txt.hbs +4 -0
- package/dist/emitter/python-templates/server.py.hbs +77 -0
- package/dist/emitter/site-scaffolder.d.ts +13 -0
- package/dist/emitter/site-scaffolder.js +70 -0
- package/dist/emitter/site-template-loader.d.ts +5 -0
- package/dist/emitter/site-template-loader.js +47 -0
- package/dist/emitter/site-templates/browser-manager.ts.hbs +233 -0
- package/dist/emitter/site-templates/config.ts.hbs +28 -0
- package/dist/emitter/site-templates/dockerfile.hbs +31 -0
- package/dist/emitter/site-templates/env.example.hbs +19 -0
- package/dist/emitter/site-templates/package.json.hbs +26 -0
- package/dist/emitter/site-templates/server-main-http.ts.hbs +108 -0
- package/dist/emitter/site-templates/server-main.ts.hbs +23 -0
- package/dist/emitter/site-templates/tool-handler-action.ts.hbs +86 -0
- package/dist/emitter/site-templates/tool-handler-form.ts.hbs +116 -0
- package/dist/emitter/site-templates/tool-handler-lifecycle.ts.hbs +146 -0
- package/dist/emitter/site-templates/tool-index.ts.hbs +11 -0
- package/dist/emitter/template-loader.d.ts +1 -0
- package/dist/emitter/template-loader.js +27 -0
- package/dist/emitter/templates/auth-provider.ts.hbs +57 -0
- package/dist/emitter/templates/config.ts.hbs +63 -0
- package/dist/emitter/templates/discovery.ts.hbs +301 -0
- package/dist/emitter/templates/dockerfile.hbs +34 -0
- package/dist/emitter/templates/env.example.hbs +28 -0
- package/dist/emitter/templates/gitignore.hbs +5 -0
- package/dist/emitter/templates/http-executor.ts.hbs +117 -0
- package/dist/emitter/templates/oauth.ts.hbs +188 -0
- package/dist/emitter/templates/package.json.hbs +25 -0
- package/dist/emitter/templates/prompts.ts.hbs +22 -0
- package/dist/emitter/templates/readme.md.hbs +123 -0
- package/dist/emitter/templates/resources.ts.hbs +63 -0
- package/dist/emitter/templates/server-main-http.ts.hbs +407 -0
- package/dist/emitter/templates/server-main.ts.hbs +40 -0
- package/dist/emitter/templates/task-handlers.ts.hbs +189 -0
- package/dist/emitter/templates/task-manager.ts.hbs +139 -0
- package/dist/emitter/templates/task-sse.ts.hbs +105 -0
- package/dist/emitter/templates/tool-handler.ts.hbs +124 -0
- package/dist/emitter/templates/tool-index.ts.hbs +11 -0
- package/dist/emitter/templates/tool-test.ts.hbs +57 -0
- package/dist/emitter/templates/trace.ts.hbs +79 -0
- package/dist/emitter/templates/tsconfig.json.hbs +16 -0
- package/dist/emitter/templates/types.ts.hbs +5 -0
- package/dist/emitter/worker-template-loader.d.ts +5 -0
- package/dist/emitter/worker-template-loader.js +33 -0
- package/dist/emitter/worker-templates/config.ts.hbs +54 -0
- package/dist/emitter/worker-templates/dev-vars.example.hbs +10 -0
- package/dist/emitter/worker-templates/gitignore.hbs +6 -0
- package/dist/emitter/worker-templates/package.json.hbs +24 -0
- package/dist/emitter/worker-templates/readme.md.hbs +53 -0
- package/dist/emitter/worker-templates/server.test.ts.hbs +20 -0
- package/dist/emitter/worker-templates/tool-handler.ts.hbs +85 -0
- package/dist/emitter/worker-templates/tool-index.ts.hbs +28 -0
- package/dist/emitter/worker-templates/tsconfig.json.hbs +17 -0
- package/dist/emitter/worker-templates/worker.ts.hbs +242 -0
- package/dist/emitter/worker-templates/wrangler.toml.hbs +19 -0
- package/dist/generator/spec-generator.d.ts +6 -0
- package/dist/generator/spec-generator.js +50 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +64 -0
- package/dist/parser/har-filter.d.ts +8 -0
- package/dist/parser/har-filter.js +71 -0
- package/dist/parser/har-loader.d.ts +2 -0
- package/dist/parser/har-loader.js +14 -0
- package/dist/parser/har-normalizer.d.ts +20 -0
- package/dist/parser/har-normalizer.js +78 -0
- package/dist/parser/index.d.ts +10 -0
- package/dist/parser/index.js +6 -0
- package/dist/parser/openapi-loader.d.ts +6 -0
- package/dist/parser/openapi-loader.js +308 -0
- package/dist/parser/operation-extractor.d.ts +13 -0
- package/dist/parser/operation-extractor.js +155 -0
- package/dist/parser/overlay-loader.d.ts +10 -0
- package/dist/parser/overlay-loader.js +184 -0
- package/dist/parser/postman-loader.d.ts +9 -0
- package/dist/parser/postman-loader.js +106 -0
- package/dist/parser/schema-converter.d.ts +12 -0
- package/dist/parser/schema-converter.js +117 -0
- package/dist/plugins/adapter.d.ts +40 -0
- package/dist/plugins/adapter.js +15 -0
- package/dist/plugins/loader.d.ts +25 -0
- package/dist/plugins/loader.js +58 -0
- package/dist/pricing.d.ts +55 -0
- package/dist/pricing.js +133 -0
- package/dist/providers/index.d.ts +15 -0
- package/dist/providers/index.js +56 -0
- package/dist/recorder/browser-recorder.d.ts +22 -0
- package/dist/recorder/browser-recorder.js +205 -0
- package/dist/registry/official-registry.d.ts +90 -0
- package/dist/registry/official-registry.js +129 -0
- package/dist/rescan/diff-engine.d.ts +5 -0
- package/dist/rescan/diff-engine.js +312 -0
- package/dist/rescan/index.d.ts +3 -0
- package/dist/rescan/index.js +2 -0
- package/dist/rescan/rescan-runner.d.ts +42 -0
- package/dist/rescan/rescan-runner.js +69 -0
- package/dist/rescan/rescan-scheduler.d.ts +41 -0
- package/dist/rescan/rescan-scheduler.js +179 -0
- package/dist/site-transformer/browser-tools.d.ts +10 -0
- package/dist/site-transformer/browser-tools.js +59 -0
- package/dist/site-transformer/index.d.ts +2 -0
- package/dist/site-transformer/index.js +2 -0
- package/dist/site-transformer/selector-healer.d.ts +8 -0
- package/dist/site-transformer/selector-healer.js +106 -0
- package/dist/site-transformer/tool-generator.d.ts +13 -0
- package/dist/site-transformer/tool-generator.js +245 -0
- package/dist/transformer/auth-detector.d.ts +13 -0
- package/dist/transformer/auth-detector.js +90 -0
- package/dist/transformer/catalog-builder.d.ts +18 -0
- package/dist/transformer/catalog-builder.js +56 -0
- package/dist/transformer/client-compat.d.ts +6 -0
- package/dist/transformer/client-compat.js +44 -0
- package/dist/transformer/har-clusterer.d.ts +9 -0
- package/dist/transformer/har-clusterer.js +27 -0
- package/dist/transformer/har-dedup.d.ts +10 -0
- package/dist/transformer/har-dedup.js +81 -0
- package/dist/transformer/har-schema-inferrer.d.ts +15 -0
- package/dist/transformer/har-schema-inferrer.js +90 -0
- package/dist/transformer/har-to-operations.d.ts +13 -0
- package/dist/transformer/har-to-operations.js +192 -0
- package/dist/transformer/index.d.ts +8 -0
- package/dist/transformer/index.js +6 -0
- package/dist/transformer/llm-namer.d.ts +6 -0
- package/dist/transformer/llm-namer.js +59 -0
- package/dist/transformer/naming.d.ts +4 -0
- package/dist/transformer/naming.js +30 -0
- package/dist/transformer/operation-filter.d.ts +13 -0
- package/dist/transformer/operation-filter.js +52 -0
- package/dist/transformer/resource-builder.d.ts +12 -0
- package/dist/transformer/resource-builder.js +80 -0
- package/dist/transformer/schema-merger.d.ts +14 -0
- package/dist/transformer/schema-merger.js +65 -0
- package/dist/transformer/tool-builder.d.ts +3 -0
- package/dist/transformer/tool-builder.js +114 -0
- package/dist/types/index.d.ts +131 -0
- package/dist/types/index.js +1 -0
- package/dist/types/site.d.ts +284 -0
- package/dist/types/site.js +8 -0
- package/dist/utils/fail.d.ts +48 -0
- package/dist/utils/fail.js +204 -0
- package/dist/utils/fs.d.ts +5 -0
- package/dist/utils/fs.js +28 -0
- package/dist/utils/interactive.d.ts +6 -0
- package/dist/utils/interactive.js +30 -0
- package/dist/utils/logger.d.ts +1 -0
- package/dist/utils/logger.js +2 -0
- package/dist/utils/sanitize.d.ts +28 -0
- package/dist/utils/sanitize.js +44 -0
- package/dist/utils/watcher.d.ts +11 -0
- package/dist/utils/watcher.js +36 -0
- package/package.json +65 -0
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Crawls a website using Playwright, visiting pages breadth-first
|
|
3
|
+
* and extracting interactive elements from each page.
|
|
4
|
+
*/
|
|
5
|
+
import { chromium } from 'playwright';
|
|
6
|
+
import { parsePage } from './dom-parser.js';
|
|
7
|
+
import { captureViewportScreenshot } from './screenshot-capture.js';
|
|
8
|
+
import { logger } from '../utils/logger.js';
|
|
9
|
+
import crypto from 'node:crypto';
|
|
10
|
+
const DEFAULT_DEPTH = 2;
|
|
11
|
+
const DEFAULT_MAX_PAGES = 20;
|
|
12
|
+
const DEFAULT_TIMEOUT = 5 * 60 * 1000;
|
|
13
|
+
const DEFAULT_VIEWPORT = { width: 1280, height: 720 };
|
|
14
|
+
/**
|
|
15
|
+
* Crawl a website and return a SiteDescriptor with all discovered pages.
|
|
16
|
+
*/
|
|
17
|
+
export async function crawlSite(options) {
|
|
18
|
+
const depth = options.depth ?? DEFAULT_DEPTH;
|
|
19
|
+
const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES;
|
|
20
|
+
const timeout = options.timeout ?? DEFAULT_TIMEOUT;
|
|
21
|
+
const viewport = options.viewport ?? DEFAULT_VIEWPORT;
|
|
22
|
+
const captureScreenshots = options.captureScreenshots ?? true;
|
|
23
|
+
const parsedUrl = new URL(options.url);
|
|
24
|
+
if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
|
|
25
|
+
throw new Error('Only http/https URLs are supported');
|
|
26
|
+
}
|
|
27
|
+
const baseUrl = `${parsedUrl.protocol}//${parsedUrl.host}`;
|
|
28
|
+
const captureHar = options.captureHar ?? false;
|
|
29
|
+
const pages = [];
|
|
30
|
+
const screenshots = new Map();
|
|
31
|
+
const harEntries = [];
|
|
32
|
+
const pendingRequests = new Map();
|
|
33
|
+
const visited = new Set();
|
|
34
|
+
const queue = [
|
|
35
|
+
{ url: options.url, currentDepth: 0 },
|
|
36
|
+
];
|
|
37
|
+
let browser;
|
|
38
|
+
try {
|
|
39
|
+
browser = await chromium.launch({ headless: options.headless ?? false });
|
|
40
|
+
const context = await browser.newContext({ viewport });
|
|
41
|
+
const page = await context.newPage();
|
|
42
|
+
// Optionally capture network requests as HAR entries during crawl
|
|
43
|
+
if (captureHar) {
|
|
44
|
+
page.on('request', (request) => {
|
|
45
|
+
pendingRequests.set(request, { startTime: Date.now() });
|
|
46
|
+
});
|
|
47
|
+
page.on('response', async (response) => {
|
|
48
|
+
const request = response.request();
|
|
49
|
+
const pending = pendingRequests.get(request);
|
|
50
|
+
if (!pending)
|
|
51
|
+
return;
|
|
52
|
+
pendingRequests.delete(request);
|
|
53
|
+
try {
|
|
54
|
+
const entry = await buildCrawlHarEntry(request, response, pending.startTime);
|
|
55
|
+
harEntries.push(entry);
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
// Some responses can't be read (redirects, aborted)
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
logger.info(`Crawling site: ${options.url} (depth: ${depth}, max pages: ${maxPages})`);
|
|
63
|
+
let lastActivityTime = Date.now();
|
|
64
|
+
while (queue.length > 0 && pages.length < maxPages) {
|
|
65
|
+
const item = queue.shift();
|
|
66
|
+
const normalizedUrl = normalizeUrl(item.url);
|
|
67
|
+
// Skip if already visited or different origin
|
|
68
|
+
if (visited.has(normalizedUrl))
|
|
69
|
+
continue;
|
|
70
|
+
if (!item.url.startsWith(baseUrl))
|
|
71
|
+
continue;
|
|
72
|
+
visited.add(normalizedUrl);
|
|
73
|
+
// Check idle timeout
|
|
74
|
+
if (Date.now() - lastActivityTime > timeout) {
|
|
75
|
+
logger.warn('Idle timeout reached during crawl');
|
|
76
|
+
break;
|
|
77
|
+
}
|
|
78
|
+
try {
|
|
79
|
+
logger.info(`[${pages.length + 1}/${maxPages}] Visiting: ${item.url}`);
|
|
80
|
+
await page.goto(item.url, {
|
|
81
|
+
waitUntil: 'domcontentloaded',
|
|
82
|
+
timeout: 15_000,
|
|
83
|
+
});
|
|
84
|
+
// Wait briefly for dynamic content to render
|
|
85
|
+
await page.waitForTimeout(1000);
|
|
86
|
+
lastActivityTime = Date.now();
|
|
87
|
+
// Parse the page DOM
|
|
88
|
+
const pageDescriptor = await parsePage(page);
|
|
89
|
+
pageDescriptor.url = item.url;
|
|
90
|
+
// Capture screenshot if enabled
|
|
91
|
+
if (captureScreenshots) {
|
|
92
|
+
const screenshot = await captureViewportScreenshot(page);
|
|
93
|
+
pageDescriptor.screenshotHash = screenshot.hash;
|
|
94
|
+
screenshots.set(pageDescriptor.pageId, screenshot.data);
|
|
95
|
+
}
|
|
96
|
+
pages.push(pageDescriptor);
|
|
97
|
+
// Queue navigation links for further crawling
|
|
98
|
+
if (item.currentDepth < depth) {
|
|
99
|
+
for (const link of pageDescriptor.links) {
|
|
100
|
+
if (link.isNavigation && !visited.has(normalizeUrl(link.href))) {
|
|
101
|
+
queue.push({ url: link.href, currentDepth: item.currentDepth + 1 });
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
catch (err) {
|
|
107
|
+
logger.warn(`Failed to crawl ${item.url}: ${err}`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
// Close browser
|
|
111
|
+
await browser.close().catch(() => { });
|
|
112
|
+
browser = undefined;
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
if (browser)
|
|
116
|
+
await browser.close().catch(() => { });
|
|
117
|
+
throw error;
|
|
118
|
+
}
|
|
119
|
+
logger.info(`Crawl complete: ${pages.length} pages discovered`);
|
|
120
|
+
// Extract site metadata from the first page
|
|
121
|
+
const metadata = {
|
|
122
|
+
title: pages[0]?.title,
|
|
123
|
+
description: undefined,
|
|
124
|
+
favicon: undefined,
|
|
125
|
+
};
|
|
126
|
+
const siteDescriptor = {
|
|
127
|
+
siteId: generateSiteId(baseUrl),
|
|
128
|
+
baseUrl,
|
|
129
|
+
pages,
|
|
130
|
+
analyzedAt: new Date().toISOString(),
|
|
131
|
+
version: 1,
|
|
132
|
+
crawlDepth: depth,
|
|
133
|
+
metadata,
|
|
134
|
+
};
|
|
135
|
+
return {
|
|
136
|
+
siteDescriptor,
|
|
137
|
+
screenshots,
|
|
138
|
+
...(captureHar ? { harEntries } : {}),
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
// ─── HAR Capture (for hybrid mode) ─────────────────────────────────
|
|
142
|
+
const MAX_RESPONSE_BODY_BYTES = 5 * 1024 * 1024;
|
|
143
|
+
const SKIP_BODY_MIME_TYPES = ['image/', 'video/', 'audio/', 'font/', 'application/octet-stream'];
|
|
144
|
+
async function buildCrawlHarEntry(request, response, startTime) {
|
|
145
|
+
const elapsed = Date.now() - startTime;
|
|
146
|
+
const url = request.url();
|
|
147
|
+
const parsedUrl = new URL(url);
|
|
148
|
+
const requestHeaders = Object.entries(request.headers()).map(([name, value]) => ({
|
|
149
|
+
name,
|
|
150
|
+
value,
|
|
151
|
+
}));
|
|
152
|
+
const queryString = [...parsedUrl.searchParams.entries()].map(([name, value]) => ({
|
|
153
|
+
name,
|
|
154
|
+
value,
|
|
155
|
+
}));
|
|
156
|
+
const postData = request.postData();
|
|
157
|
+
const contentTypeHeader = request.headers()['content-type'] ?? '';
|
|
158
|
+
const responseHeaders = Object.entries(response.headers()).map(([name, value]) => ({
|
|
159
|
+
name,
|
|
160
|
+
value,
|
|
161
|
+
}));
|
|
162
|
+
let responseText;
|
|
163
|
+
const responseMimeType = response.headers()['content-type'] ?? '';
|
|
164
|
+
const skipBody = SKIP_BODY_MIME_TYPES.some((m) => responseMimeType.startsWith(m));
|
|
165
|
+
const contentLength = parseInt(response.headers()['content-length'] ?? '0', 10);
|
|
166
|
+
if (!skipBody && contentLength <= MAX_RESPONSE_BODY_BYTES) {
|
|
167
|
+
try {
|
|
168
|
+
const body = await response.body();
|
|
169
|
+
if (body.length <= MAX_RESPONSE_BODY_BYTES) {
|
|
170
|
+
responseText = body.toString('utf-8');
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
catch {
|
|
174
|
+
// Body may not be available
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return {
|
|
178
|
+
startedDateTime: new Date(startTime).toISOString(),
|
|
179
|
+
time: elapsed,
|
|
180
|
+
request: {
|
|
181
|
+
method: request.method(),
|
|
182
|
+
url,
|
|
183
|
+
httpVersion: 'HTTP/1.1',
|
|
184
|
+
headers: requestHeaders,
|
|
185
|
+
queryString,
|
|
186
|
+
cookies: [],
|
|
187
|
+
headersSize: -1,
|
|
188
|
+
bodySize: postData ? Buffer.byteLength(postData) : 0,
|
|
189
|
+
...(postData
|
|
190
|
+
? {
|
|
191
|
+
postData: {
|
|
192
|
+
mimeType: contentTypeHeader.split(';')[0].trim() || 'application/octet-stream',
|
|
193
|
+
text: postData,
|
|
194
|
+
},
|
|
195
|
+
}
|
|
196
|
+
: {}),
|
|
197
|
+
},
|
|
198
|
+
response: {
|
|
199
|
+
status: response.status(),
|
|
200
|
+
statusText: response.statusText(),
|
|
201
|
+
httpVersion: 'HTTP/1.1',
|
|
202
|
+
headers: responseHeaders,
|
|
203
|
+
cookies: [],
|
|
204
|
+
content: {
|
|
205
|
+
size: responseText ? Buffer.byteLength(responseText) : 0,
|
|
206
|
+
mimeType: responseMimeType.split(';')[0].trim() || 'application/octet-stream',
|
|
207
|
+
...(responseText ? { text: responseText } : {}),
|
|
208
|
+
},
|
|
209
|
+
redirectURL: '',
|
|
210
|
+
headersSize: -1,
|
|
211
|
+
bodySize: responseText ? Buffer.byteLength(responseText) : 0,
|
|
212
|
+
},
|
|
213
|
+
cache: {},
|
|
214
|
+
timings: {
|
|
215
|
+
send: 1,
|
|
216
|
+
wait: Math.max(1, elapsed - 2),
|
|
217
|
+
receive: 1,
|
|
218
|
+
},
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
// ─── Helpers ────────────────────────────────────────────────────────
|
|
222
|
+
function normalizeUrl(url) {
|
|
223
|
+
try {
|
|
224
|
+
const parsed = new URL(url);
|
|
225
|
+
// Remove trailing slash, fragment, and normalize
|
|
226
|
+
return `${parsed.origin}${parsed.pathname.replace(/\/$/, '')}${parsed.search}`;
|
|
227
|
+
}
|
|
228
|
+
catch {
|
|
229
|
+
return url;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
function generateSiteId(baseUrl) {
|
|
233
|
+
const hash = crypto.createHash('sha256').update(baseUrl).digest('hex').slice(0, 12);
|
|
234
|
+
return `site_${hash}`;
|
|
235
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
export type BillingPlan = 'free' | 'hobbyist' | 'pro' | 'team' | 'enterprise';
|
|
2
|
+
export interface PlanLimits {
|
|
3
|
+
/** Maximum allowed usage in minutes per billing period */
|
|
4
|
+
maxMinutes: number;
|
|
5
|
+
plan: BillingPlan;
|
|
6
|
+
}
|
|
7
|
+
export interface QuotaCheckResult {
|
|
8
|
+
allowed: boolean;
|
|
9
|
+
/** Minutes remaining in the current period */
|
|
10
|
+
remaining: number;
|
|
11
|
+
/** Plan limit in minutes */
|
|
12
|
+
limit: number;
|
|
13
|
+
}
|
|
14
|
+
/** Get the minute limits for a given billing plan. */
|
|
15
|
+
export declare function getPlanLimits(plan: BillingPlan): PlanLimits;
|
|
16
|
+
/** Monthly tool-call limit for a plan. */
|
|
17
|
+
export declare function getCallLimit(plan: BillingPlan): number;
|
|
18
|
+
/** Maximum hosted servers for a plan. */
|
|
19
|
+
export declare function getServerLimit(plan: BillingPlan): number;
|
|
20
|
+
/**
|
|
21
|
+
* Check whether a user may create another server given how many they already
|
|
22
|
+
* have. `allowed` is false once the count reaches the plan's server limit.
|
|
23
|
+
*/
|
|
24
|
+
export declare function checkServerLimit(plan: BillingPlan, currentCount: number): {
|
|
25
|
+
allowed: boolean;
|
|
26
|
+
limit: number;
|
|
27
|
+
remaining: number;
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* Check whether a user has tool-call quota remaining for the billing period.
|
|
31
|
+
*/
|
|
32
|
+
export declare function checkCallQuota(plan: BillingPlan, usedCalls: number): {
|
|
33
|
+
allowed: boolean;
|
|
34
|
+
limit: number;
|
|
35
|
+
remaining: number;
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Check whether a user has quota remaining under their plan.
|
|
39
|
+
*
|
|
40
|
+
* @param _userId - The user ID (reserved for future per-user overrides)
|
|
41
|
+
* @param plan - The user's billing plan
|
|
42
|
+
* @param currentUsageMs - Current usage in milliseconds for the billing period
|
|
43
|
+
*/
|
|
44
|
+
export declare function checkQuota(_userId: string, plan: BillingPlan, currentUsageMs: number): QuotaCheckResult;
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
const PLAN_LIMITS = {
|
|
2
|
+
free: 500,
|
|
3
|
+
hobbyist: 5_000,
|
|
4
|
+
pro: 50_000,
|
|
5
|
+
team: 500_000,
|
|
6
|
+
enterprise: Infinity,
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Monthly tool-call (request) limits per plan. Overage on paid plans is billed
|
|
10
|
+
* separately; the free tier is a hard cap. Tunable business constants.
|
|
11
|
+
*/
|
|
12
|
+
const PLAN_CALL_LIMITS = {
|
|
13
|
+
free: 1_000,
|
|
14
|
+
hobbyist: 10_000,
|
|
15
|
+
pro: 100_000,
|
|
16
|
+
team: 1_000_000,
|
|
17
|
+
enterprise: Infinity,
|
|
18
|
+
};
|
|
19
|
+
/** Maximum number of concurrently hosted servers per plan. */
|
|
20
|
+
const PLAN_SERVER_LIMITS = {
|
|
21
|
+
free: 1,
|
|
22
|
+
hobbyist: 3,
|
|
23
|
+
pro: 5,
|
|
24
|
+
team: 20,
|
|
25
|
+
enterprise: Infinity,
|
|
26
|
+
};
|
|
27
|
+
/** Get the minute limits for a given billing plan. */
|
|
28
|
+
export function getPlanLimits(plan) {
|
|
29
|
+
return {
|
|
30
|
+
maxMinutes: PLAN_LIMITS[plan],
|
|
31
|
+
plan,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
/** Monthly tool-call limit for a plan. */
|
|
35
|
+
export function getCallLimit(plan) {
|
|
36
|
+
return PLAN_CALL_LIMITS[plan] ?? PLAN_CALL_LIMITS.free;
|
|
37
|
+
}
|
|
38
|
+
/** Maximum hosted servers for a plan. */
|
|
39
|
+
export function getServerLimit(plan) {
|
|
40
|
+
return PLAN_SERVER_LIMITS[plan] ?? PLAN_SERVER_LIMITS.free;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Check whether a user may create another server given how many they already
|
|
44
|
+
* have. `allowed` is false once the count reaches the plan's server limit.
|
|
45
|
+
*/
|
|
46
|
+
export function checkServerLimit(plan, currentCount) {
|
|
47
|
+
const limit = getServerLimit(plan);
|
|
48
|
+
return {
|
|
49
|
+
allowed: currentCount < limit,
|
|
50
|
+
limit,
|
|
51
|
+
remaining: Math.max(0, limit - currentCount),
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Check whether a user has tool-call quota remaining for the billing period.
|
|
56
|
+
*/
|
|
57
|
+
export function checkCallQuota(plan, usedCalls) {
|
|
58
|
+
const limit = getCallLimit(plan);
|
|
59
|
+
return {
|
|
60
|
+
allowed: usedCalls < limit,
|
|
61
|
+
limit,
|
|
62
|
+
remaining: Math.max(0, limit - usedCalls),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Check whether a user has quota remaining under their plan.
|
|
67
|
+
*
|
|
68
|
+
* @param _userId - The user ID (reserved for future per-user overrides)
|
|
69
|
+
* @param plan - The user's billing plan
|
|
70
|
+
* @param currentUsageMs - Current usage in milliseconds for the billing period
|
|
71
|
+
*/
|
|
72
|
+
export function checkQuota(_userId, plan, currentUsageMs) {
|
|
73
|
+
const limit = PLAN_LIMITS[plan];
|
|
74
|
+
const usedMinutes = currentUsageMs / 60_000;
|
|
75
|
+
const remaining = Math.max(0, limit - usedMinutes);
|
|
76
|
+
return {
|
|
77
|
+
allowed: usedMinutes < limit,
|
|
78
|
+
remaining,
|
|
79
|
+
limit,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Promotional tool-call credits.
|
|
3
|
+
*
|
|
4
|
+
* A persistent per-user balance of bonus tool-calls, consumed when a user is
|
|
5
|
+
* over their plan's monthly call quota (before the 429 hard cap). Admins grant
|
|
6
|
+
* and revoke from the dashboard; the serving hot path consumes.
|
|
7
|
+
*
|
|
8
|
+
* Concurrency: `consume` MUST be atomic — two concurrent over-quota requests
|
|
9
|
+
* must not both decrement from the same read value (lost update = leaked free
|
|
10
|
+
* credits). The Pg implementation does the decrement in a single conditional
|
|
11
|
+
* UPDATE; the in-memory one is safe because each call runs to completion within
|
|
12
|
+
* one event-loop turn.
|
|
13
|
+
*
|
|
14
|
+
* The ledger records only admin grants/revokes (low volume, truly auditable).
|
|
15
|
+
* Per-request consumption is intentionally NOT ledgered — that would add a write
|
|
16
|
+
* per over-quota tool call on the hot path; consumption totals are derivable
|
|
17
|
+
* from usage accounting.
|
|
18
|
+
*/
|
|
19
|
+
import type { Database } from '../db/index.js';
|
|
20
|
+
export interface ConsumeResult {
|
|
21
|
+
consumed: boolean;
|
|
22
|
+
/** Remaining balance after the operation (best-effort; 0 when not consumed). */
|
|
23
|
+
balance: number;
|
|
24
|
+
}
|
|
25
|
+
export interface LedgerEntry {
|
|
26
|
+
delta: number;
|
|
27
|
+
reason: string;
|
|
28
|
+
actorId: string | null;
|
|
29
|
+
balanceAfter: number | null;
|
|
30
|
+
createdAt: string;
|
|
31
|
+
}
|
|
32
|
+
export interface CreditStore {
|
|
33
|
+
balance(userId: string): Promise<number>;
|
|
34
|
+
/** Add `amount` credits. Returns the new balance. */
|
|
35
|
+
grant(userId: string, amount: number, actorId: string): Promise<number>;
|
|
36
|
+
/** Remove up to `amount` credits (never below zero). Returns the new balance. */
|
|
37
|
+
revoke(userId: string, amount: number, actorId: string): Promise<number>;
|
|
38
|
+
/** Atomically consume `amount` credits if the balance covers it (all-or-nothing). */
|
|
39
|
+
consume(userId: string, amount: number): Promise<ConsumeResult>;
|
|
40
|
+
recentLedger(userId: string, limit: number): Promise<LedgerEntry[]>;
|
|
41
|
+
}
|
|
42
|
+
export declare class InMemoryCreditStore implements CreditStore {
|
|
43
|
+
private balances;
|
|
44
|
+
private ledger;
|
|
45
|
+
/** Set a balance, bounding the map so a long-lived dev process can't grow it
|
|
46
|
+
* without limit (drops the oldest-inserted half when the cap is exceeded). */
|
|
47
|
+
private setBalance;
|
|
48
|
+
private push;
|
|
49
|
+
balance(userId: string): Promise<number>;
|
|
50
|
+
grant(userId: string, amount: number, actorId: string): Promise<number>;
|
|
51
|
+
revoke(userId: string, amount: number, actorId: string): Promise<number>;
|
|
52
|
+
consume(userId: string, amount: number): Promise<ConsumeResult>;
|
|
53
|
+
recentLedger(userId: string, limit: number): Promise<LedgerEntry[]>;
|
|
54
|
+
}
|
|
55
|
+
export declare class PgCreditStore implements CreditStore {
|
|
56
|
+
private db;
|
|
57
|
+
constructor(db: Database);
|
|
58
|
+
private writeLedger;
|
|
59
|
+
balance(userId: string): Promise<number>;
|
|
60
|
+
grant(userId: string, amount: number, actorId: string): Promise<number>;
|
|
61
|
+
revoke(userId: string, amount: number, actorId: string): Promise<number>;
|
|
62
|
+
consume(userId: string, amount: number): Promise<ConsumeResult>;
|
|
63
|
+
recentLedger(userId: string, limit: number): Promise<LedgerEntry[]>;
|
|
64
|
+
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Promotional tool-call credits.
|
|
3
|
+
*
|
|
4
|
+
* A persistent per-user balance of bonus tool-calls, consumed when a user is
|
|
5
|
+
* over their plan's monthly call quota (before the 429 hard cap). Admins grant
|
|
6
|
+
* and revoke from the dashboard; the serving hot path consumes.
|
|
7
|
+
*
|
|
8
|
+
* Concurrency: `consume` MUST be atomic — two concurrent over-quota requests
|
|
9
|
+
* must not both decrement from the same read value (lost update = leaked free
|
|
10
|
+
* credits). The Pg implementation does the decrement in a single conditional
|
|
11
|
+
* UPDATE; the in-memory one is safe because each call runs to completion within
|
|
12
|
+
* one event-loop turn.
|
|
13
|
+
*
|
|
14
|
+
* The ledger records only admin grants/revokes (low volume, truly auditable).
|
|
15
|
+
* Per-request consumption is intentionally NOT ledgered — that would add a write
|
|
16
|
+
* per over-quota tool call on the hot path; consumption totals are derivable
|
|
17
|
+
* from usage accounting.
|
|
18
|
+
*/
|
|
19
|
+
import { randomBytes } from 'node:crypto';
|
|
20
|
+
// ---------------------------------------------------------------------------
|
|
21
|
+
// In-memory (dev / no database)
|
|
22
|
+
// ---------------------------------------------------------------------------
|
|
23
|
+
const MAX_INMEMORY_LEDGER = 5_000;
|
|
24
|
+
/** Bound the per-user balance map (dev/no-DB only; prod uses the users table). */
|
|
25
|
+
const MAX_INMEMORY_BALANCES = 50_000;
|
|
26
|
+
export class InMemoryCreditStore {
|
|
27
|
+
balances = new Map();
|
|
28
|
+
ledger = [];
|
|
29
|
+
/** Set a balance, bounding the map so a long-lived dev process can't grow it
|
|
30
|
+
* without limit (drops the oldest-inserted half when the cap is exceeded). */
|
|
31
|
+
setBalance(userId, value) {
|
|
32
|
+
if (!this.balances.has(userId) && this.balances.size >= MAX_INMEMORY_BALANCES) {
|
|
33
|
+
const keep = Math.floor(MAX_INMEMORY_BALANCES / 2);
|
|
34
|
+
this.balances = new Map([...this.balances].slice(-keep));
|
|
35
|
+
}
|
|
36
|
+
this.balances.set(userId, value);
|
|
37
|
+
}
|
|
38
|
+
push(entry) {
|
|
39
|
+
this.ledger.push(entry);
|
|
40
|
+
if (this.ledger.length > MAX_INMEMORY_LEDGER) {
|
|
41
|
+
this.ledger.splice(0, this.ledger.length - MAX_INMEMORY_LEDGER);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
async balance(userId) {
|
|
45
|
+
return this.balances.get(userId) ?? 0;
|
|
46
|
+
}
|
|
47
|
+
async grant(userId, amount, actorId) {
|
|
48
|
+
const next = (this.balances.get(userId) ?? 0) + amount;
|
|
49
|
+
this.setBalance(userId, next);
|
|
50
|
+
this.push({
|
|
51
|
+
userId,
|
|
52
|
+
delta: amount,
|
|
53
|
+
reason: 'admin_grant',
|
|
54
|
+
actorId,
|
|
55
|
+
balanceAfter: next,
|
|
56
|
+
createdAt: new Date().toISOString(),
|
|
57
|
+
});
|
|
58
|
+
return next;
|
|
59
|
+
}
|
|
60
|
+
async revoke(userId, amount, actorId) {
|
|
61
|
+
const current = this.balances.get(userId) ?? 0;
|
|
62
|
+
const next = Math.max(0, current - amount);
|
|
63
|
+
this.setBalance(userId, next);
|
|
64
|
+
this.push({
|
|
65
|
+
userId,
|
|
66
|
+
delta: next - current,
|
|
67
|
+
reason: 'admin_revoke',
|
|
68
|
+
actorId,
|
|
69
|
+
balanceAfter: next,
|
|
70
|
+
createdAt: new Date().toISOString(),
|
|
71
|
+
});
|
|
72
|
+
return next;
|
|
73
|
+
}
|
|
74
|
+
async consume(userId, amount) {
|
|
75
|
+
const current = this.balances.get(userId) ?? 0;
|
|
76
|
+
if (amount <= 0 || current < amount)
|
|
77
|
+
return { consumed: false, balance: 0 };
|
|
78
|
+
const next = current - amount;
|
|
79
|
+
this.setBalance(userId, next);
|
|
80
|
+
return { consumed: true, balance: next };
|
|
81
|
+
}
|
|
82
|
+
async recentLedger(userId, limit) {
|
|
83
|
+
return this.ledger
|
|
84
|
+
.filter((e) => e.userId === userId)
|
|
85
|
+
.slice(-limit)
|
|
86
|
+
.reverse()
|
|
87
|
+
.map(({ userId: _u, ...rest }) => rest);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
// ---------------------------------------------------------------------------
|
|
91
|
+
// Postgres
|
|
92
|
+
// ---------------------------------------------------------------------------
|
|
93
|
+
export class PgCreditStore {
|
|
94
|
+
db;
|
|
95
|
+
constructor(db) {
|
|
96
|
+
this.db = db;
|
|
97
|
+
}
|
|
98
|
+
async writeLedger(userId, delta, reason, actorId, balanceAfter) {
|
|
99
|
+
await this.db.query(`INSERT INTO credit_ledger (id, user_id, delta, reason, actor_id, balance_after)
|
|
100
|
+
VALUES ($1, $2, $3, $4, $5, $6)`, [randomBytes(16).toString('hex'), userId, delta, reason, actorId, balanceAfter]);
|
|
101
|
+
}
|
|
102
|
+
async balance(userId) {
|
|
103
|
+
const { rows } = await this.db.query('SELECT credits_calls FROM users WHERE id = $1', [userId]);
|
|
104
|
+
const v = rows[0]?.credits_calls ?? 0;
|
|
105
|
+
return typeof v === 'number' ? v : Number(v) || 0;
|
|
106
|
+
}
|
|
107
|
+
async grant(userId, amount, actorId) {
|
|
108
|
+
const { rows } = await this.db.query('UPDATE users SET credits_calls = credits_calls + $2 WHERE id = $1 RETURNING credits_calls', [userId, amount]);
|
|
109
|
+
if (rows.length === 0)
|
|
110
|
+
throw new Error(`User "${userId}" not found`);
|
|
111
|
+
const next = typeof rows[0].credits_calls === 'number'
|
|
112
|
+
? rows[0].credits_calls
|
|
113
|
+
: Number(rows[0].credits_calls);
|
|
114
|
+
await this.writeLedger(userId, amount, 'admin_grant', actorId, next);
|
|
115
|
+
return next;
|
|
116
|
+
}
|
|
117
|
+
async revoke(userId, amount, actorId) {
|
|
118
|
+
// Single atomic statement: lock the row (FOR UPDATE), clamp at zero, and
|
|
119
|
+
// return both the old and new balance so the ledger delta is exact even
|
|
120
|
+
// under concurrent admin revokes (no read-modify-write window).
|
|
121
|
+
const { rows } = await this.db.query(`WITH prev AS (
|
|
122
|
+
SELECT credits_calls AS old_balance FROM users WHERE id = $1 FOR UPDATE
|
|
123
|
+
)
|
|
124
|
+
UPDATE users
|
|
125
|
+
SET credits_calls = GREATEST(0, users.credits_calls - $2)
|
|
126
|
+
FROM prev
|
|
127
|
+
WHERE users.id = $1
|
|
128
|
+
RETURNING users.credits_calls AS new_balance, prev.old_balance`, [userId, amount]);
|
|
129
|
+
if (rows.length === 0)
|
|
130
|
+
throw new Error(`User "${userId}" not found`);
|
|
131
|
+
const toNum = (v) => (typeof v === 'number' ? v : Number(v));
|
|
132
|
+
const next = toNum(rows[0].new_balance);
|
|
133
|
+
const before = toNum(rows[0].old_balance);
|
|
134
|
+
// Record the actual delta applied (clamped at zero), not the requested amount.
|
|
135
|
+
await this.writeLedger(userId, next - before, 'admin_revoke', actorId, next);
|
|
136
|
+
return next;
|
|
137
|
+
}
|
|
138
|
+
async consume(userId, amount) {
|
|
139
|
+
if (amount <= 0)
|
|
140
|
+
return { consumed: false, balance: 0 };
|
|
141
|
+
// Single atomic statement: decrement only if the balance covers it.
|
|
142
|
+
const { rows } = await this.db.query('UPDATE users SET credits_calls = credits_calls - $2 WHERE id = $1 AND credits_calls >= $2 RETURNING credits_calls', [userId, amount]);
|
|
143
|
+
if (rows.length === 0)
|
|
144
|
+
return { consumed: false, balance: 0 };
|
|
145
|
+
const balance = typeof rows[0].credits_calls === 'number'
|
|
146
|
+
? rows[0].credits_calls
|
|
147
|
+
: Number(rows[0].credits_calls);
|
|
148
|
+
return { consumed: true, balance };
|
|
149
|
+
}
|
|
150
|
+
async recentLedger(userId, limit) {
|
|
151
|
+
const { rows } = await this.db.query(`SELECT delta, reason, actor_id, balance_after, created_at
|
|
152
|
+
FROM credit_ledger WHERE user_id = $1 ORDER BY created_at DESC LIMIT $2`, [userId, limit]);
|
|
153
|
+
return rows.map((row) => {
|
|
154
|
+
const createdAt = row.created_at;
|
|
155
|
+
return {
|
|
156
|
+
delta: typeof row.delta === 'number' ? row.delta : Number(row.delta ?? 0),
|
|
157
|
+
reason: String(row.reason ?? ''),
|
|
158
|
+
actorId: row.actor_id != null ? String(row.actor_id) : null,
|
|
159
|
+
balanceAfter: row.balance_after != null
|
|
160
|
+
? typeof row.balance_after === 'number'
|
|
161
|
+
? row.balance_after
|
|
162
|
+
: Number(row.balance_after)
|
|
163
|
+
: null,
|
|
164
|
+
createdAt: typeof createdAt === 'string' ? createdAt : new Date(createdAt).toISOString(),
|
|
165
|
+
};
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { UsageTracker } from './usage-tracker.js';
|
|
2
|
+
export type { UsageEvent, UsageEventType, UsageSummary } from './usage-tracker.js';
|
|
3
|
+
export { checkQuota, getPlanLimits } from './billing-engine.js';
|
|
4
|
+
export type { BillingPlan, PlanLimits, QuotaCheckResult } from './billing-engine.js';
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable per-user usage accounting tied to a billing period.
|
|
3
|
+
*
|
|
4
|
+
* Records tool calls (and total requests) for the served MCP traffic so that
|
|
5
|
+
* quota can be enforced and usage survives process restarts. Backed by the
|
|
6
|
+
* `usage_summaries` Postgres table when a database is configured; otherwise an
|
|
7
|
+
* in-memory fallback is used (dev / single-process).
|
|
8
|
+
*/
|
|
9
|
+
import type { Database } from '../db/index.js';
|
|
10
|
+
export interface BillingPeriod {
|
|
11
|
+
/** ISO timestamp of the period start (inclusive). */
|
|
12
|
+
start: string;
|
|
13
|
+
/** ISO timestamp of the period end (exclusive). */
|
|
14
|
+
end: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Compute the billing period containing `now`.
|
|
18
|
+
*
|
|
19
|
+
* When `anchorEndMs` (a Stripe subscription's current_period_end) is provided
|
|
20
|
+
* and in the future, the period is the month ending at that anchor; otherwise
|
|
21
|
+
* it falls back to the calendar month (UTC). Calendar-month alignment is a safe
|
|
22
|
+
* default for the free tier and approximates monthly subscriptions closely.
|
|
23
|
+
*/
|
|
24
|
+
export declare function getBillingPeriod(now?: Date, anchorEndMs?: number): BillingPeriod;
|
|
25
|
+
export interface UsageStore {
|
|
26
|
+
/** Add tool-call and request counts for a user in the given period. */
|
|
27
|
+
record(userId: string, period: BillingPeriod, toolCalls: number, requests: number): Promise<void>;
|
|
28
|
+
/** Total tool calls a user has made in the given period. */
|
|
29
|
+
getToolCalls(userId: string, period: BillingPeriod): Promise<number>;
|
|
30
|
+
}
|
|
31
|
+
export declare class InMemoryUsageStore implements UsageStore {
|
|
32
|
+
private counts;
|
|
33
|
+
private key;
|
|
34
|
+
record(userId: string, period: BillingPeriod, toolCalls: number, requests: number): Promise<void>;
|
|
35
|
+
getToolCalls(userId: string, period: BillingPeriod): Promise<number>;
|
|
36
|
+
}
|
|
37
|
+
export declare class PgUsageStore implements UsageStore {
|
|
38
|
+
private db;
|
|
39
|
+
constructor(db: Database);
|
|
40
|
+
record(userId: string, period: BillingPeriod, toolCalls: number, requests: number): Promise<void>;
|
|
41
|
+
getToolCalls(userId: string, period: BillingPeriod): Promise<number>;
|
|
42
|
+
}
|