mcard-js 2.1.2 → 2.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -1
- package/dist/ptr/llm/Config.d.ts.map +1 -1
- package/dist/ptr/llm/Config.js +16 -0
- package/dist/ptr/llm/Config.js.map +1 -1
- package/dist/ptr/llm/LLMRuntime.d.ts.map +1 -1
- package/dist/ptr/llm/LLMRuntime.js +8 -0
- package/dist/ptr/llm/LLMRuntime.js.map +1 -1
- package/dist/ptr/llm/providers/MLCLLMProvider.d.ts +22 -0
- package/dist/ptr/llm/providers/MLCLLMProvider.d.ts.map +1 -0
- package/dist/ptr/llm/providers/MLCLLMProvider.js +155 -0
- package/dist/ptr/llm/providers/MLCLLMProvider.js.map +1 -0
- package/dist/ptr/llm/providers/WebLLMProvider.d.ts +22 -0
- package/dist/ptr/llm/providers/WebLLMProvider.d.ts.map +1 -0
- package/dist/ptr/llm/providers/WebLLMProvider.js +151 -0
- package/dist/ptr/llm/providers/WebLLMProvider.js.map +1 -0
- package/dist/ptr/node/CLMRunner.d.ts +3 -1
- package/dist/ptr/node/CLMRunner.d.ts.map +1 -1
- package/dist/ptr/node/CLMRunner.js +33 -1
- package/dist/ptr/node/CLMRunner.js.map +1 -1
- package/dist/ptr/node/NetworkConfig.d.ts +155 -0
- package/dist/ptr/node/NetworkConfig.d.ts.map +1 -0
- package/dist/ptr/node/NetworkConfig.js +8 -0
- package/dist/ptr/node/NetworkConfig.js.map +1 -0
- package/dist/ptr/node/NetworkRuntime.d.ts +115 -0
- package/dist/ptr/node/NetworkRuntime.d.ts.map +1 -0
- package/dist/ptr/node/NetworkRuntime.js +792 -0
- package/dist/ptr/node/NetworkRuntime.js.map +1 -0
- package/dist/ptr/node/Runtimes.d.ts.map +1 -1
- package/dist/ptr/node/Runtimes.js +10 -0
- package/dist/ptr/node/Runtimes.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,792 @@
|
|
|
1
|
+
import * as http from 'http';
|
|
2
|
+
import { MCard } from '../../model/MCard.js';
|
|
3
|
+
/**
|
|
4
|
+
* Network Runtime for handling declarative network operations.
|
|
5
|
+
*
|
|
6
|
+
* Supports builtins:
|
|
7
|
+
* - http_request (general)
|
|
8
|
+
* - http_get
|
|
9
|
+
* - http_post
|
|
10
|
+
* - load_url (enhanced)
|
|
11
|
+
* - mcard_send (send MCard to remote)
|
|
12
|
+
* - listen_http (receive MCards)
|
|
13
|
+
* - mcard_sync (synchronize collection)
|
|
14
|
+
* - listen_sync (listen for sync requests)
|
|
15
|
+
*
|
|
16
|
+
* Security:
|
|
17
|
+
* Configure via environment variables:
|
|
18
|
+
* - CLM_ALLOWED_DOMAINS: Comma-separated allowed domains (e.g., "api.example.com,*.trusted.com")
|
|
19
|
+
* - CLM_BLOCKED_DOMAINS: Comma-separated blocked domains (takes precedence)
|
|
20
|
+
* - CLM_BLOCK_LOCALHOST: Set to "true" to block localhost
|
|
21
|
+
* - CLM_BLOCK_PRIVATE_IPS: Set to "true" to block private IP ranges
|
|
22
|
+
*/
|
|
23
|
+
export class NetworkRuntime {
|
|
24
|
+
collection;
|
|
25
|
+
securityConfig;
|
|
26
|
+
responseCache;
|
|
27
|
+
rateLimiter;
|
|
28
|
+
defaultRateLimit = { tokensPerSecond: 10, maxBurst: 20 };
|
|
29
|
+
constructor(collection) {
|
|
30
|
+
this.collection = collection;
|
|
31
|
+
this.securityConfig = this.loadSecurityConfigFromEnv();
|
|
32
|
+
this.responseCache = new Map();
|
|
33
|
+
this.rateLimiter = new Map();
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Load security configuration from environment variables
|
|
37
|
+
*/
|
|
38
|
+
loadSecurityConfigFromEnv() {
|
|
39
|
+
const parseList = (value) => {
|
|
40
|
+
if (!value)
|
|
41
|
+
return undefined;
|
|
42
|
+
return value.split(',').map(s => s.trim()).filter(s => s.length > 0);
|
|
43
|
+
};
|
|
44
|
+
return {
|
|
45
|
+
allowed_domains: parseList(process.env.CLM_ALLOWED_DOMAINS),
|
|
46
|
+
blocked_domains: parseList(process.env.CLM_BLOCKED_DOMAINS),
|
|
47
|
+
allowed_protocols: parseList(process.env.CLM_ALLOWED_PROTOCOLS),
|
|
48
|
+
block_private_ips: process.env.CLM_BLOCK_PRIVATE_IPS === 'true',
|
|
49
|
+
block_localhost: process.env.CLM_BLOCK_LOCALHOST === 'true'
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Validate URL against security policy
|
|
54
|
+
* Throws SecurityViolationError if URL is not allowed
|
|
55
|
+
*/
|
|
56
|
+
validateUrlSecurity(urlString) {
|
|
57
|
+
let url;
|
|
58
|
+
try {
|
|
59
|
+
url = new URL(urlString);
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
throw this.createSecurityError('DOMAIN_BLOCKED', `Invalid URL: ${urlString}`, urlString);
|
|
63
|
+
}
|
|
64
|
+
const hostname = url.hostname.toLowerCase();
|
|
65
|
+
const protocol = url.protocol.replace(':', '');
|
|
66
|
+
// 1. Check blocked domains (takes precedence)
|
|
67
|
+
if (this.securityConfig.blocked_domains) {
|
|
68
|
+
for (const pattern of this.securityConfig.blocked_domains) {
|
|
69
|
+
if (this.matchDomainPattern(hostname, pattern)) {
|
|
70
|
+
throw this.createSecurityError('DOMAIN_BLOCKED', `Domain '${hostname}' is blocked by security policy`, urlString);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
// 2. Check allowed domains (if configured, only these are allowed)
|
|
75
|
+
if (this.securityConfig.allowed_domains && this.securityConfig.allowed_domains.length > 0) {
|
|
76
|
+
const isAllowed = this.securityConfig.allowed_domains.some(pattern => this.matchDomainPattern(hostname, pattern));
|
|
77
|
+
if (!isAllowed) {
|
|
78
|
+
throw this.createSecurityError('DOMAIN_NOT_ALLOWED', `Domain '${hostname}' is not in the allowed list`, urlString);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
// 3. Check allowed protocols
|
|
82
|
+
if (this.securityConfig.allowed_protocols && this.securityConfig.allowed_protocols.length > 0) {
|
|
83
|
+
if (!this.securityConfig.allowed_protocols.includes(protocol)) {
|
|
84
|
+
throw this.createSecurityError('PROTOCOL_NOT_ALLOWED', `Protocol '${protocol}' is not allowed. Allowed: ${this.securityConfig.allowed_protocols.join(', ')}`, urlString);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
// 4. Check localhost blocking
|
|
88
|
+
if (this.securityConfig.block_localhost) {
|
|
89
|
+
if (hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '::1') {
|
|
90
|
+
throw this.createSecurityError('LOCALHOST_BLOCKED', 'Localhost access is blocked by security policy', urlString);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
// 5. Check private IP blocking
|
|
94
|
+
if (this.securityConfig.block_private_ips) {
|
|
95
|
+
if (this.isPrivateIP(hostname)) {
|
|
96
|
+
throw this.createSecurityError('PRIVATE_IP_BLOCKED', `Private IP '${hostname}' is blocked by security policy`, urlString);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Match hostname against domain pattern (supports wildcards like *.example.com)
|
|
102
|
+
*/
|
|
103
|
+
matchDomainPattern(hostname, pattern) {
|
|
104
|
+
const patternLower = pattern.toLowerCase();
|
|
105
|
+
if (patternLower.startsWith('*.')) {
|
|
106
|
+
// Wildcard pattern: *.example.com matches sub.example.com, a.b.example.com
|
|
107
|
+
const suffix = patternLower.slice(1); // .example.com
|
|
108
|
+
return hostname.endsWith(suffix) || hostname === patternLower.slice(2);
|
|
109
|
+
}
|
|
110
|
+
return hostname === patternLower;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Check if hostname is a private IP address
|
|
114
|
+
*/
|
|
115
|
+
isPrivateIP(hostname) {
|
|
116
|
+
// Simple regex checks for common private IP ranges
|
|
117
|
+
const privatePatterns = [
|
|
118
|
+
/^10\.\d+\.\d+\.\d+$/, // 10.x.x.x
|
|
119
|
+
/^192\.168\.\d+\.\d+$/, // 192.168.x.x
|
|
120
|
+
/^172\.(1[6-9]|2\d|3[01])\.\d+\.\d+$/, // 172.16-31.x.x
|
|
121
|
+
/^169\.254\.\d+\.\d+$/, // Link-local
|
|
122
|
+
/^fc00:/i, // IPv6 private
|
|
123
|
+
/^fd00:/i, // IPv6 private
|
|
124
|
+
];
|
|
125
|
+
return privatePatterns.some(pattern => pattern.test(hostname));
|
|
126
|
+
}
|
|
127
|
+
createSecurityError(code, message, url) {
|
|
128
|
+
const error = new Error(message);
|
|
129
|
+
error.securityViolation = { code, message, url };
|
|
130
|
+
return error;
|
|
131
|
+
}
|
|
132
|
+
// ============ MCard Serialization Helpers ============
|
|
133
|
+
/**
|
|
134
|
+
* Serialize an MCard to a JSON-safe payload for network transfer
|
|
135
|
+
*/
|
|
136
|
+
serializeMCard(card) {
|
|
137
|
+
return {
|
|
138
|
+
hash: card.hash,
|
|
139
|
+
content: Buffer.from(card.content).toString('base64'),
|
|
140
|
+
g_time: card.g_time,
|
|
141
|
+
contentType: card.contentType,
|
|
142
|
+
hashFunction: card.hashFunction
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Deserialize a JSON payload back to an MCard
|
|
147
|
+
* Uses fromData if hash/g_time provided (preserves identity)
|
|
148
|
+
* Otherwise creates new MCard (generates new hash/g_time)
|
|
149
|
+
*/
|
|
150
|
+
async deserializeMCard(json) {
|
|
151
|
+
if (!json.content) {
|
|
152
|
+
throw new Error('Missing content in MCard payload');
|
|
153
|
+
}
|
|
154
|
+
const content = Buffer.from(json.content, 'base64');
|
|
155
|
+
if (json.hash && json.g_time) {
|
|
156
|
+
return MCard.fromData(content, json.hash, json.g_time);
|
|
157
|
+
}
|
|
158
|
+
return MCard.create(content);
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Verify hash matches content (optional strict mode)
|
|
162
|
+
*/
|
|
163
|
+
verifyMCardHash(card, expectedHash) {
|
|
164
|
+
if (card.hash !== expectedHash) {
|
|
165
|
+
console.warn(`[Network] Hash mismatch. Expected: ${expectedHash}, Got: ${card.hash}`);
|
|
166
|
+
return false;
|
|
167
|
+
}
|
|
168
|
+
return true;
|
|
169
|
+
}
|
|
170
|
+
// ============ Retry Logic ============
|
|
171
|
+
/**
|
|
172
|
+
* Calculate delay for retry attempt based on backoff strategy
|
|
173
|
+
*/
|
|
174
|
+
calculateBackoffDelay(attempt, strategy, baseDelay, maxDelay) {
|
|
175
|
+
let delay;
|
|
176
|
+
switch (strategy) {
|
|
177
|
+
case 'exponential':
|
|
178
|
+
delay = baseDelay * Math.pow(2, attempt - 1);
|
|
179
|
+
break;
|
|
180
|
+
case 'linear':
|
|
181
|
+
delay = baseDelay * attempt;
|
|
182
|
+
break;
|
|
183
|
+
case 'constant':
|
|
184
|
+
default:
|
|
185
|
+
delay = baseDelay;
|
|
186
|
+
}
|
|
187
|
+
// Add jitter (±10%) to prevent thundering herd
|
|
188
|
+
const jitter = delay * 0.1 * (Math.random() * 2 - 1);
|
|
189
|
+
delay = Math.round(delay + jitter);
|
|
190
|
+
return maxDelay ? Math.min(delay, maxDelay) : delay;
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Check if HTTP status code should trigger a retry
|
|
194
|
+
*/
|
|
195
|
+
shouldRetryStatus(status, retryOn) {
|
|
196
|
+
const defaultRetryStatuses = [408, 429, 500, 502, 503, 504];
|
|
197
|
+
const retryStatuses = retryOn || defaultRetryStatuses;
|
|
198
|
+
return retryStatuses.includes(status);
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Sleep for a given duration
|
|
202
|
+
*/
|
|
203
|
+
sleep(ms) {
|
|
204
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
205
|
+
}
|
|
206
|
+
// ============ Response Caching ============
|
|
207
|
+
/**
|
|
208
|
+
* Generate cache key from request config
|
|
209
|
+
*/
|
|
210
|
+
generateCacheKey(method, url, body) {
|
|
211
|
+
const keyData = `${method}:${url}:${body || ''}`;
|
|
212
|
+
// Simple hash for cache key (not cryptographic)
|
|
213
|
+
let hash = 0;
|
|
214
|
+
for (let i = 0; i < keyData.length; i++) {
|
|
215
|
+
const char = keyData.charCodeAt(i);
|
|
216
|
+
hash = ((hash << 5) - hash) + char;
|
|
217
|
+
hash = hash & hash;
|
|
218
|
+
}
|
|
219
|
+
return `cache_${Math.abs(hash).toString(36)}`;
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Get cached response if valid
|
|
223
|
+
*/
|
|
224
|
+
getCachedResponse(cacheKey) {
|
|
225
|
+
const cached = this.responseCache.get(cacheKey);
|
|
226
|
+
if (cached && cached.expiresAt > Date.now()) {
|
|
227
|
+
return { ...cached.response, timing: { ...cached.response.timing, total: 0 } };
|
|
228
|
+
}
|
|
229
|
+
// Clean up expired entry
|
|
230
|
+
if (cached) {
|
|
231
|
+
this.responseCache.delete(cacheKey);
|
|
232
|
+
}
|
|
233
|
+
return null;
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Cache a response with TTL
|
|
237
|
+
*/
|
|
238
|
+
cacheResponse(cacheKey, response, ttlSeconds) {
|
|
239
|
+
this.responseCache.set(cacheKey, {
|
|
240
|
+
response,
|
|
241
|
+
expiresAt: Date.now() + (ttlSeconds * 1000)
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Store response in MCard collection for persistent caching
|
|
246
|
+
*/
|
|
247
|
+
async cacheToPersistentStorage(cacheKey, response, ttlSeconds) {
|
|
248
|
+
if (!this.collection)
|
|
249
|
+
return;
|
|
250
|
+
const cacheEntry = {
|
|
251
|
+
key: cacheKey,
|
|
252
|
+
response,
|
|
253
|
+
expiresAt: Date.now() + (ttlSeconds * 1000),
|
|
254
|
+
cachedAt: new Date().toISOString()
|
|
255
|
+
};
|
|
256
|
+
const card = await MCard.create(JSON.stringify(cacheEntry));
|
|
257
|
+
await this.collection.add(card);
|
|
258
|
+
}
|
|
259
|
+
// ============ Rate Limiting ============
|
|
260
|
+
/**
|
|
261
|
+
* Token bucket rate limiter
|
|
262
|
+
* Returns true if request should proceed, false if rate limited
|
|
263
|
+
*/
|
|
264
|
+
checkRateLimit(domain) {
|
|
265
|
+
const now = Date.now();
|
|
266
|
+
const bucket = this.rateLimiter.get(domain) || {
|
|
267
|
+
tokens: this.defaultRateLimit.maxBurst,
|
|
268
|
+
lastRefill: now
|
|
269
|
+
};
|
|
270
|
+
// Refill tokens based on time elapsed
|
|
271
|
+
const elapsed = (now - bucket.lastRefill) / 1000;
|
|
272
|
+
const refill = elapsed * this.defaultRateLimit.tokensPerSecond;
|
|
273
|
+
bucket.tokens = Math.min(this.defaultRateLimit.maxBurst, bucket.tokens + refill);
|
|
274
|
+
bucket.lastRefill = now;
|
|
275
|
+
if (bucket.tokens >= 1) {
|
|
276
|
+
bucket.tokens -= 1;
|
|
277
|
+
this.rateLimiter.set(domain, bucket);
|
|
278
|
+
return true;
|
|
279
|
+
}
|
|
280
|
+
this.rateLimiter.set(domain, bucket);
|
|
281
|
+
return false;
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* Wait until rate limit allows request
|
|
285
|
+
*/
|
|
286
|
+
async waitForRateLimit(domain) {
|
|
287
|
+
while (!this.checkRateLimit(domain)) {
|
|
288
|
+
await this.sleep(100);
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
async execute(_code, context, config, _chapterDir) {
|
|
292
|
+
const builtin = config.builtin;
|
|
293
|
+
if (!builtin) {
|
|
294
|
+
throw new Error('NetworkRuntime requires "builtin" to be defined in config.');
|
|
295
|
+
}
|
|
296
|
+
switch (builtin) {
|
|
297
|
+
case 'http_request':
|
|
298
|
+
return this.handleHttpRequest(config.config || {}, context);
|
|
299
|
+
case 'http_get':
|
|
300
|
+
return this.handleHttpGet(config.config || {}, context);
|
|
301
|
+
case 'http_post':
|
|
302
|
+
return this.handleHttpPost(config.config || {}, context);
|
|
303
|
+
case 'load_url':
|
|
304
|
+
return this.handleLoadUrl(config.config || {}, context);
|
|
305
|
+
case 'mcard_send':
|
|
306
|
+
return this.handleMCardSend(config.config || {}, context);
|
|
307
|
+
case 'listen_http':
|
|
308
|
+
return this.handleListenHttp(config.config || {}, context);
|
|
309
|
+
case 'mcard_sync':
|
|
310
|
+
return this.handleMCardSync(config.config || {}, context);
|
|
311
|
+
case 'listen_sync':
|
|
312
|
+
return this.handleListenSync(config.config || {}, context);
|
|
313
|
+
default:
|
|
314
|
+
throw new Error(`Unknown network builtin: ${builtin}`);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
async handleHttpGet(config, context) {
|
|
318
|
+
// http_get is just http_request with method=GET
|
|
319
|
+
return this.handleHttpRequest({ ...config, method: 'GET' }, context);
|
|
320
|
+
}
|
|
321
|
+
async handleHttpPost(config, context) {
|
|
322
|
+
// http_post is just http_request with method=POST and body handling
|
|
323
|
+
const params = { ...config, method: 'POST' };
|
|
324
|
+
if (config.json) {
|
|
325
|
+
params.headers = { ...params.headers, 'Content-Type': 'application/json' };
|
|
326
|
+
params.body = JSON.stringify(config.json);
|
|
327
|
+
}
|
|
328
|
+
return this.handleHttpRequest(params, context);
|
|
329
|
+
}
|
|
330
|
+
async handleHttpRequest(config, context) {
|
|
331
|
+
const startTime = Date.now();
|
|
332
|
+
// 1. Interpolate variables in URL, headers, body
|
|
333
|
+
const url = this.interpolate(config.url, context);
|
|
334
|
+
// 1b. Security check: Validate URL against policy
|
|
335
|
+
this.validateUrlSecurity(url);
|
|
336
|
+
const method = config.method || 'GET';
|
|
337
|
+
const headers = this.interpolateHeaders(config.headers || {}, context);
|
|
338
|
+
let body = config.body;
|
|
339
|
+
// If body is defined and interpolation is needed
|
|
340
|
+
if (typeof body === 'string') {
|
|
341
|
+
body = this.interpolate(body, context);
|
|
342
|
+
}
|
|
343
|
+
else if (typeof body === 'object' && body !== null) {
|
|
344
|
+
// For JSON bodies passed as objects
|
|
345
|
+
body = JSON.stringify(body);
|
|
346
|
+
}
|
|
347
|
+
// Add Query Params
|
|
348
|
+
const fetchUrl = new URL(url);
|
|
349
|
+
if (config.query_params) {
|
|
350
|
+
for (const [key, value] of Object.entries(config.query_params)) {
|
|
351
|
+
fetchUrl.searchParams.append(key, this.interpolate(String(value), context));
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
// 2. Check cache first (only for GET requests)
|
|
355
|
+
const cacheConfig = config.cache;
|
|
356
|
+
const cacheKey = this.generateCacheKey(method, fetchUrl.toString(), body);
|
|
357
|
+
if (cacheConfig?.enabled && method === 'GET') {
|
|
358
|
+
const cachedResponse = this.getCachedResponse(cacheKey);
|
|
359
|
+
if (cachedResponse) {
|
|
360
|
+
console.log(`[Network] Cache hit for ${url}`);
|
|
361
|
+
return { ...cachedResponse, cached: true };
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
// 3. Rate limiting
|
|
365
|
+
const domain = fetchUrl.hostname;
|
|
366
|
+
await this.waitForRateLimit(domain);
|
|
367
|
+
// 4. Retry configuration
|
|
368
|
+
const retryConfig = config.retry || {
|
|
369
|
+
max_attempts: 1,
|
|
370
|
+
backoff: 'exponential',
|
|
371
|
+
base_delay: 1000,
|
|
372
|
+
max_delay: 30000
|
|
373
|
+
};
|
|
374
|
+
let lastError = null;
|
|
375
|
+
let lastStatus = null;
|
|
376
|
+
let retriesAttempted = 0;
|
|
377
|
+
for (let attempt = 1; attempt <= retryConfig.max_attempts; attempt++) {
|
|
378
|
+
const timeout = typeof config.timeout === 'number'
|
|
379
|
+
? config.timeout
|
|
380
|
+
: config.timeout?.total || 30000;
|
|
381
|
+
const controller = new AbortController();
|
|
382
|
+
const timeoutId = setTimeout(() => controller.abort(), timeout);
|
|
383
|
+
try {
|
|
384
|
+
const ttfbStart = Date.now();
|
|
385
|
+
const response = await fetch(fetchUrl.toString(), {
|
|
386
|
+
method,
|
|
387
|
+
headers,
|
|
388
|
+
body: body,
|
|
389
|
+
signal: controller.signal
|
|
390
|
+
});
|
|
391
|
+
clearTimeout(timeoutId);
|
|
392
|
+
// Check if we should retry based on status
|
|
393
|
+
if (!response.ok && this.shouldRetryStatus(response.status, retryConfig.retry_on)) {
|
|
394
|
+
lastStatus = response.status;
|
|
395
|
+
if (attempt < retryConfig.max_attempts) {
|
|
396
|
+
retriesAttempted++;
|
|
397
|
+
const delay = this.calculateBackoffDelay(attempt, retryConfig.backoff, retryConfig.base_delay, retryConfig.max_delay);
|
|
398
|
+
console.log(`[Network] Retry ${attempt}/${retryConfig.max_attempts} for ${url} (status: ${response.status}, delay: ${delay}ms)`);
|
|
399
|
+
await this.sleep(delay);
|
|
400
|
+
continue;
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
// 5. Process Response
|
|
404
|
+
const ttfbTime = Date.now() - ttfbStart;
|
|
405
|
+
const responseType = config.response_type || 'json';
|
|
406
|
+
let responseBody;
|
|
407
|
+
if (responseType === 'json') {
|
|
408
|
+
try {
|
|
409
|
+
responseBody = await response.json();
|
|
410
|
+
}
|
|
411
|
+
catch {
|
|
412
|
+
responseBody = await response.text(); // Fallback
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
else if (responseType === 'text') {
|
|
416
|
+
responseBody = await response.text();
|
|
417
|
+
}
|
|
418
|
+
else if (responseType === 'binary') {
|
|
419
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
420
|
+
responseBody = Buffer.from(arrayBuffer).toString('base64');
|
|
421
|
+
}
|
|
422
|
+
else {
|
|
423
|
+
responseBody = await response.text();
|
|
424
|
+
}
|
|
425
|
+
const totalTime = Date.now() - startTime;
|
|
426
|
+
// 6. Calculate mcard_hash for content-addressed response
|
|
427
|
+
let mcard_hash;
|
|
428
|
+
try {
|
|
429
|
+
const bodyStr = typeof responseBody === 'string'
|
|
430
|
+
? responseBody
|
|
431
|
+
: JSON.stringify(responseBody);
|
|
432
|
+
const responseCard = await MCard.create(bodyStr);
|
|
433
|
+
mcard_hash = responseCard.hash;
|
|
434
|
+
}
|
|
435
|
+
catch {
|
|
436
|
+
// Non-critical: skip hash if creation fails
|
|
437
|
+
}
|
|
438
|
+
const timing = {
|
|
439
|
+
dns: 0,
|
|
440
|
+
connect: 0,
|
|
441
|
+
ttfb: ttfbTime,
|
|
442
|
+
total: totalTime
|
|
443
|
+
};
|
|
444
|
+
const result = {
|
|
445
|
+
success: true,
|
|
446
|
+
status: response.status,
|
|
447
|
+
headers: Object.fromEntries(response.headers.entries()),
|
|
448
|
+
body: responseBody,
|
|
449
|
+
timing,
|
|
450
|
+
mcard_hash
|
|
451
|
+
};
|
|
452
|
+
// 7. Cache successful GET responses
|
|
453
|
+
if (cacheConfig?.enabled && method === 'GET' && response.ok) {
|
|
454
|
+
this.cacheResponse(cacheKey, result, cacheConfig.ttl);
|
|
455
|
+
if (cacheConfig.storage === 'mcard') {
|
|
456
|
+
await this.cacheToPersistentStorage(cacheKey, result, cacheConfig.ttl);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
return result;
|
|
460
|
+
}
|
|
461
|
+
catch (error) {
|
|
462
|
+
clearTimeout(timeoutId);
|
|
463
|
+
lastError = error;
|
|
464
|
+
// Only retry on network errors or timeouts
|
|
465
|
+
if (attempt < retryConfig.max_attempts) {
|
|
466
|
+
retriesAttempted++;
|
|
467
|
+
const delay = this.calculateBackoffDelay(attempt, retryConfig.backoff, retryConfig.base_delay, retryConfig.max_delay);
|
|
468
|
+
console.log(`[Network] Retry ${attempt}/${retryConfig.max_attempts} for ${url} (error: ${lastError.message}, delay: ${delay}ms)`);
|
|
469
|
+
await this.sleep(delay);
|
|
470
|
+
continue;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
// All retries exhausted
|
|
475
|
+
const err = lastError;
|
|
476
|
+
return {
|
|
477
|
+
success: false,
|
|
478
|
+
error: {
|
|
479
|
+
code: err?.name === 'AbortError' ? 'TIMEOUT' : 'HTTP_ERROR',
|
|
480
|
+
message: err?.message || 'Request failed after retries',
|
|
481
|
+
status: lastStatus,
|
|
482
|
+
retries_attempted: retriesAttempted
|
|
483
|
+
}
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
async handleLoadUrl(config, context) {
|
|
487
|
+
// Basic implementation of load_url using fetch
|
|
488
|
+
const url = this.interpolate(config.url, context);
|
|
489
|
+
// Security check
|
|
490
|
+
this.validateUrlSecurity(url);
|
|
491
|
+
try {
|
|
492
|
+
// Basic fetch, more advanced extraction logic (selector, cleaning)
|
|
493
|
+
// would require cheerio or jsdom (omitted for minimal dependency)
|
|
494
|
+
const res = await fetch(url);
|
|
495
|
+
const text = await res.text();
|
|
496
|
+
// Simple extraction placeholder
|
|
497
|
+
// Real implementation would use parsed extraction config
|
|
498
|
+
return {
|
|
499
|
+
url,
|
|
500
|
+
content: text, // Raw content for now
|
|
501
|
+
status: res.status,
|
|
502
|
+
headers: Object.fromEntries(res.headers.entries())
|
|
503
|
+
};
|
|
504
|
+
}
|
|
505
|
+
catch (e) {
|
|
506
|
+
return { success: false, error: String(e) };
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
async handleMCardSend(config, context) {
|
|
510
|
+
if (!this.collection) {
|
|
511
|
+
throw new Error('MCard Send requires a CardCollection.');
|
|
512
|
+
}
|
|
513
|
+
const hash = this.interpolate(config.hash, context);
|
|
514
|
+
const url = this.interpolate(config.url, context);
|
|
515
|
+
const card = await this.collection.get(hash);
|
|
516
|
+
if (!card) {
|
|
517
|
+
return { success: false, error: `MCard not found: ${hash}` };
|
|
518
|
+
}
|
|
519
|
+
// Use helper for consistent serialization
|
|
520
|
+
const payload = this.serializeMCard(card);
|
|
521
|
+
const response = await this.handleHttpPost({
|
|
522
|
+
url,
|
|
523
|
+
json: payload,
|
|
524
|
+
headers: config.headers
|
|
525
|
+
}, context);
|
|
526
|
+
return response;
|
|
527
|
+
}
|
|
528
|
+
async handleListenHttp(config, context) {
|
|
529
|
+
const port = Number(this.interpolate(String(config.port || 3000), context));
|
|
530
|
+
const path = this.interpolate(config.path || '/mcard', context);
|
|
531
|
+
return new Promise((resolve, reject) => {
|
|
532
|
+
const server = http.createServer(async (req, res) => {
|
|
533
|
+
if (req.method === 'POST' && req.url === path) {
|
|
534
|
+
const bodyChunks = [];
|
|
535
|
+
req.on('data', chunk => bodyChunks.push(chunk));
|
|
536
|
+
req.on('end', async () => {
|
|
537
|
+
try {
|
|
538
|
+
const body = Buffer.concat(bodyChunks).toString();
|
|
539
|
+
const json = JSON.parse(body);
|
|
540
|
+
// Use helper for consistent deserialization
|
|
541
|
+
const card = await this.deserializeMCard(json);
|
|
542
|
+
// Verify hash if provided
|
|
543
|
+
if (json.hash) {
|
|
544
|
+
this.verifyMCardHash(card, json.hash);
|
|
545
|
+
}
|
|
546
|
+
// Store
|
|
547
|
+
if (this.collection) {
|
|
548
|
+
await this.collection.add(card);
|
|
549
|
+
}
|
|
550
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
551
|
+
res.end(JSON.stringify({ success: true, hash: card.hash }));
|
|
552
|
+
// If user configured "one_shot", we might close server here
|
|
553
|
+
// For now, keep running? Or resolve the promise?
|
|
554
|
+
// This is blocking.
|
|
555
|
+
}
|
|
556
|
+
catch (e) {
|
|
557
|
+
res.writeHead(400, { 'Content-Type': 'application/json' });
|
|
558
|
+
res.end(JSON.stringify({ success: false, error: String(e) }));
|
|
559
|
+
}
|
|
560
|
+
});
|
|
561
|
+
}
|
|
562
|
+
else {
|
|
563
|
+
res.writeHead(404);
|
|
564
|
+
res.end();
|
|
565
|
+
}
|
|
566
|
+
});
|
|
567
|
+
server.listen(port, () => {
|
|
568
|
+
console.log(`[Network] Listening on port ${port} at ${path}`);
|
|
569
|
+
// If we want to return immediately and keep server running in background:
|
|
570
|
+
resolve({
|
|
571
|
+
success: true,
|
|
572
|
+
message: `Server started on port ${port}`,
|
|
573
|
+
// We probably should return a way to stop it, but for a 1-shot execution...
|
|
574
|
+
// Or if we want to block until stopped?
|
|
575
|
+
});
|
|
576
|
+
});
|
|
577
|
+
server.on('error', (err) => {
|
|
578
|
+
reject(err);
|
|
579
|
+
});
|
|
580
|
+
});
|
|
581
|
+
}
|
|
582
|
+
async handleMCardSync(config, context) {
|
|
583
|
+
if (!this.collection) {
|
|
584
|
+
throw new Error('MCard Sync requires a CardCollection.');
|
|
585
|
+
}
|
|
586
|
+
const mode = this.interpolate(config.mode || 'pull', context);
|
|
587
|
+
const urlParams = this.interpolate(config.url, context);
|
|
588
|
+
// Ensure no trailing slash
|
|
589
|
+
const url = urlParams.endsWith('/') ? urlParams.slice(0, -1) : urlParams;
|
|
590
|
+
// 1. Get Local Manifest
|
|
591
|
+
const localCards = await this.collection.getAllMCardsRaw();
|
|
592
|
+
const localHashes = new Set(localCards.map(c => c.hash));
|
|
593
|
+
// 2. Get Remote Manifest
|
|
594
|
+
const manifestRes = await this.handleHttpRequest({
|
|
595
|
+
url: `${url}/manifest`,
|
|
596
|
+
method: 'GET'
|
|
597
|
+
}, context);
|
|
598
|
+
if (!manifestRes.success) {
|
|
599
|
+
throw new Error(`Failed to fetch remote manifest: ${manifestRes.error?.message}`);
|
|
600
|
+
}
|
|
601
|
+
const remoteHashes = new Set(manifestRes.body);
|
|
602
|
+
const stats = {
|
|
603
|
+
mode,
|
|
604
|
+
local_total: localHashes.size,
|
|
605
|
+
remote_total: remoteHashes.size,
|
|
606
|
+
synced: 0
|
|
607
|
+
};
|
|
608
|
+
// Helper: Push cards to remote
|
|
609
|
+
const pushCards = async () => {
|
|
610
|
+
const toSend = [];
|
|
611
|
+
for (const card of localCards) {
|
|
612
|
+
if (!remoteHashes.has(card.hash)) {
|
|
613
|
+
toSend.push(card);
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
if (toSend.length > 0) {
|
|
617
|
+
const payload = {
|
|
618
|
+
cards: toSend.map(card => this.serializeMCard(card))
|
|
619
|
+
};
|
|
620
|
+
const pushRes = await this.handleHttpPost({
|
|
621
|
+
url: `${url}/batch`,
|
|
622
|
+
json: payload,
|
|
623
|
+
headers: config.headers
|
|
624
|
+
}, context);
|
|
625
|
+
if (!pushRes.success) {
|
|
626
|
+
throw new Error(`Failed to push batch: ${pushRes.error?.message}`);
|
|
627
|
+
}
|
|
628
|
+
return toSend.length;
|
|
629
|
+
}
|
|
630
|
+
return 0;
|
|
631
|
+
};
|
|
632
|
+
// Helper: Pull cards from remote
|
|
633
|
+
const pullCards = async () => {
|
|
634
|
+
const neededHashes = [];
|
|
635
|
+
for (const h of remoteHashes) {
|
|
636
|
+
if (!localHashes.has(h)) {
|
|
637
|
+
neededHashes.push(h);
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
if (neededHashes.length > 0) {
|
|
641
|
+
const fetchRes = await this.handleHttpPost({
|
|
642
|
+
url: `${url}/get`,
|
|
643
|
+
json: { hashes: neededHashes },
|
|
644
|
+
headers: config.headers
|
|
645
|
+
}, context);
|
|
646
|
+
if (!fetchRes.success) {
|
|
647
|
+
throw new Error(`Failed to pull batch: ${fetchRes.error?.message}`);
|
|
648
|
+
}
|
|
649
|
+
const receivedCards = fetchRes.body.cards;
|
|
650
|
+
for (const json of receivedCards) {
|
|
651
|
+
const card = await this.deserializeMCard(json);
|
|
652
|
+
await this.collection.add(card);
|
|
653
|
+
}
|
|
654
|
+
return receivedCards.length;
|
|
655
|
+
}
|
|
656
|
+
return 0;
|
|
657
|
+
};
|
|
658
|
+
if (mode === 'push') {
|
|
659
|
+
stats.synced = await pushCards();
|
|
660
|
+
}
|
|
661
|
+
else if (mode === 'pull') {
|
|
662
|
+
stats.synced = await pullCards();
|
|
663
|
+
}
|
|
664
|
+
else if (mode === 'both' || mode === 'bidirectional') {
|
|
665
|
+
// Bidirectional: Push first, then pull
|
|
666
|
+
const pushed = await pushCards();
|
|
667
|
+
const pulled = await pullCards();
|
|
668
|
+
stats.pushed = pushed;
|
|
669
|
+
stats.pulled = pulled;
|
|
670
|
+
stats.synced = pushed + pulled;
|
|
671
|
+
}
|
|
672
|
+
else {
|
|
673
|
+
throw new Error(`Unknown sync mode: ${mode}. Valid modes: push, pull, both`);
|
|
674
|
+
}
|
|
675
|
+
return { success: true, stats };
|
|
676
|
+
}
|
|
677
|
+
async handleListenSync(config, context) {
|
|
678
|
+
if (!this.collection) {
|
|
679
|
+
throw new Error('Listen Sync requires a CardCollection.');
|
|
680
|
+
}
|
|
681
|
+
const port = Number(this.interpolate(String(config.port || 3000), context));
|
|
682
|
+
const basePath = this.interpolate(config.base_path || '/sync', context);
|
|
683
|
+
return new Promise((resolve, reject) => {
|
|
684
|
+
const server = http.createServer(async (req, res) => {
|
|
685
|
+
const url = req.url || '';
|
|
686
|
+
// Read Body helper
|
|
687
|
+
const readBody = async () => {
|
|
688
|
+
return new Promise((res, rej) => {
|
|
689
|
+
const chunks = [];
|
|
690
|
+
req.on('data', c => chunks.push(c));
|
|
691
|
+
req.on('end', () => {
|
|
692
|
+
try {
|
|
693
|
+
const str = Buffer.concat(chunks).toString();
|
|
694
|
+
res(JSON.parse(str || '{}'));
|
|
695
|
+
}
|
|
696
|
+
catch (e) {
|
|
697
|
+
rej(e);
|
|
698
|
+
}
|
|
699
|
+
});
|
|
700
|
+
req.on('error', rej);
|
|
701
|
+
});
|
|
702
|
+
};
|
|
703
|
+
try {
|
|
704
|
+
// Route: GET /manifest
|
|
705
|
+
if (req.method === 'GET' && url === `${basePath}/manifest`) {
|
|
706
|
+
const all = await this.collection.getAllMCardsRaw();
|
|
707
|
+
const hashes = all.map(c => c.hash);
|
|
708
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
709
|
+
res.end(JSON.stringify(hashes));
|
|
710
|
+
return;
|
|
711
|
+
}
|
|
712
|
+
// Route: POST /batch (Recv Push)
|
|
713
|
+
if (req.method === 'POST' && url === `${basePath}/batch`) {
|
|
714
|
+
const json = await readBody();
|
|
715
|
+
const cards = json.cards || [];
|
|
716
|
+
let added = 0;
|
|
717
|
+
for (const cJson of cards) {
|
|
718
|
+
const card = await this.deserializeMCard(cJson);
|
|
719
|
+
await this.collection.add(card);
|
|
720
|
+
added++;
|
|
721
|
+
}
|
|
722
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
723
|
+
res.end(JSON.stringify({ success: true, added }));
|
|
724
|
+
return;
|
|
725
|
+
}
|
|
726
|
+
// Route: POST /get (Serve Pull)
|
|
727
|
+
if (req.method === 'POST' && url === `${basePath}/get`) {
|
|
728
|
+
const json = await readBody();
|
|
729
|
+
const requestedHashes = json.hashes || [];
|
|
730
|
+
const foundCards = [];
|
|
731
|
+
for (const h of requestedHashes) {
|
|
732
|
+
const card = await this.collection.get(h);
|
|
733
|
+
if (card) {
|
|
734
|
+
foundCards.push(this.serializeMCard(card));
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
738
|
+
res.end(JSON.stringify({ success: true, cards: foundCards }));
|
|
739
|
+
return;
|
|
740
|
+
}
|
|
741
|
+
res.writeHead(404);
|
|
742
|
+
res.end();
|
|
743
|
+
}
|
|
744
|
+
catch (e) {
|
|
745
|
+
res.writeHead(500, { 'Content-Type': 'application/json' });
|
|
746
|
+
res.end(JSON.stringify({ success: false, error: String(e) }));
|
|
747
|
+
}
|
|
748
|
+
});
|
|
749
|
+
server.listen(port, () => {
|
|
750
|
+
console.log(`[Network] Sync listening on port ${port} at ${basePath}`);
|
|
751
|
+
resolve({
|
|
752
|
+
success: true,
|
|
753
|
+
message: `Sync Server started on port ${port}`,
|
|
754
|
+
port,
|
|
755
|
+
basePath
|
|
756
|
+
});
|
|
757
|
+
});
|
|
758
|
+
server.on('error', (err) => {
|
|
759
|
+
reject(err);
|
|
760
|
+
});
|
|
761
|
+
});
|
|
762
|
+
}
|
|
763
|
+
/**
|
|
764
|
+
* Simple variable interpolation: ${key} or ${input.key} or ${secrets.KEY}
|
|
765
|
+
* context = { input: ..., secrets: ..., ... }
|
|
766
|
+
*/
|
|
767
|
+
interpolate(text, context) {
|
|
768
|
+
if (!text || typeof text !== 'string')
|
|
769
|
+
return text;
|
|
770
|
+
return text.replace(/\$\{([^}]+)\}/g, (_, path) => {
|
|
771
|
+
const keys = path.split('.');
|
|
772
|
+
let val = context;
|
|
773
|
+
for (const key of keys) {
|
|
774
|
+
if (val && typeof val === 'object' && key in val) {
|
|
775
|
+
val = val[key];
|
|
776
|
+
}
|
|
777
|
+
else {
|
|
778
|
+
return ''; // Not found
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
return String(val);
|
|
782
|
+
});
|
|
783
|
+
}
|
|
784
|
+
interpolateHeaders(headers, context) {
|
|
785
|
+
const result = {};
|
|
786
|
+
for (const [key, val] of Object.entries(headers)) {
|
|
787
|
+
result[key] = this.interpolate(val, context);
|
|
788
|
+
}
|
|
789
|
+
return result;
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
//# sourceMappingURL=NetworkRuntime.js.map
|