lsh-framework 3.2.5 → 3.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +72 -34
  3. package/dist/commands/ipfs.js +7 -12
  4. package/dist/commands/self.js +22 -16
  5. package/dist/commands/sync.js +49 -38
  6. package/dist/constants/config.js +3 -0
  7. package/dist/lib/floating-point-arithmetic.js +2 -2
  8. package/dist/lib/ipfs-client-manager.js +51 -13
  9. package/dist/lib/ipfs-secrets-storage.js +21 -16
  10. package/dist/lib/ipfs-sync.js +88 -14
  11. package/dist/lib/secrets-manager.js +117 -47
  12. package/dist/lib/sync-key-store.js +87 -0
  13. package/dist/services/secrets/secrets.js +77 -39
  14. package/package.json +16 -16
  15. package/dist/__tests__/fixtures/job-fixtures.js +0 -204
  16. package/dist/__tests__/fixtures/supabase-mocks.js +0 -252
  17. package/dist/daemon/job-registry.js +0 -556
  18. package/dist/daemon/lshd.js +0 -968
  19. package/dist/daemon/saas-api-routes.js +0 -599
  20. package/dist/daemon/saas-api-server.js +0 -231
  21. package/dist/examples/supabase-integration.js +0 -106
  22. package/dist/lib/api-response.js +0 -226
  23. package/dist/lib/base-command-registrar.js +0 -287
  24. package/dist/lib/base-job-manager.js +0 -295
  25. package/dist/lib/cloud-config-manager.js +0 -348
  26. package/dist/lib/cron-job-manager.js +0 -368
  27. package/dist/lib/daemon-client-helper.js +0 -145
  28. package/dist/lib/daemon-client.js +0 -513
  29. package/dist/lib/database-persistence.js +0 -727
  30. package/dist/lib/database-schema.js +0 -259
  31. package/dist/lib/database-types.js +0 -90
  32. package/dist/lib/enhanced-history-system.js +0 -247
  33. package/dist/lib/history-system.js +0 -246
  34. package/dist/lib/job-manager.js +0 -436
  35. package/dist/lib/job-storage-database.js +0 -164
  36. package/dist/lib/job-storage-memory.js +0 -73
  37. package/dist/lib/local-storage-adapter.js +0 -507
  38. package/dist/lib/optimized-job-scheduler.js +0 -356
  39. package/dist/lib/saas-audit.js +0 -215
  40. package/dist/lib/saas-auth.js +0 -465
  41. package/dist/lib/saas-billing.js +0 -503
  42. package/dist/lib/saas-email.js +0 -403
  43. package/dist/lib/saas-encryption.js +0 -221
  44. package/dist/lib/saas-organizations.js +0 -662
  45. package/dist/lib/saas-secrets.js +0 -408
  46. package/dist/lib/saas-types.js +0 -165
  47. package/dist/lib/supabase-client.js +0 -125
  48. package/dist/lib/supabase-utils.js +0 -396
  49. package/dist/services/cron/cron-registrar.js +0 -240
  50. package/dist/services/cron/cron.js +0 -9
  51. package/dist/services/daemon/daemon-registrar.js +0 -585
  52. package/dist/services/daemon/daemon.js +0 -9
  53. package/dist/services/supabase/supabase-registrar.js +0 -375
  54. package/dist/services/supabase/supabase.js +0 -9
@@ -1,356 +0,0 @@
1
- /**
2
- * Optimized Job Scheduler
3
- *
4
- * A priority queue-based job scheduler that efficiently manages scheduled jobs.
5
- * Instead of linearly scanning all jobs every 2 seconds, this scheduler uses
6
- * a min-heap to only check jobs that are actually due.
7
- *
8
- * Performance Improvements:
9
- * - O(log n) job insertion/removal vs O(n) linear scan
10
- * - O(1) to check if any jobs are due
11
- * - Only processes jobs that are actually due
12
- * - Smart sleep intervals based on next job time
13
- *
14
- * @see Issue #108: PERFORMANCE: Optimize daemon job scheduling algorithm
15
- */
16
- import { MinHeap } from './min-heap.js';
17
- import { createLogger } from './logger.js';
18
- import { EventEmitter } from 'events';
19
- const DEFAULT_CONFIG = {
20
- minCheckInterval: 100,
21
- maxCheckInterval: 60000,
22
- dueBuffer: 50,
23
- debug: false,
24
- };
25
- export class OptimizedJobScheduler extends EventEmitter {
26
- heap;
27
- jobMap = new Map();
28
- config;
29
- logger = createLogger('OptimizedJobScheduler');
30
- checkTimer;
31
- isRunning = false;
32
- lastRunTimes = new Map();
33
- // Metrics tracking
34
- metrics = {
35
- totalJobs: 0,
36
- dueJobs: 0,
37
- nextCheckTime: 0,
38
- averageCheckTime: 0,
39
- memoryUsage: 0,
40
- totalChecks: 0,
41
- totalExecuted: 0,
42
- };
43
- checkTimings = [];
44
- MAX_TIMING_SAMPLES = 100;
45
- constructor(config = {}) {
46
- super();
47
- this.config = { ...DEFAULT_CONFIG, ...config };
48
- this.heap = new MinHeap(entry => entry.nextRun, entry => entry.jobId);
49
- }
50
- /**
51
- * Start the scheduler
52
- */
53
- start() {
54
- if (this.isRunning) {
55
- return;
56
- }
57
- this.isRunning = true;
58
- this.logger.info('Optimized job scheduler started');
59
- this.scheduleNextCheck();
60
- }
61
- /**
62
- * Stop the scheduler
63
- */
64
- stop() {
65
- this.isRunning = false;
66
- if (this.checkTimer) {
67
- clearTimeout(this.checkTimer);
68
- this.checkTimer = undefined;
69
- }
70
- this.logger.info('Optimized job scheduler stopped');
71
- }
72
- /**
73
- * Add a job to the scheduler
74
- * @param job The job specification
75
- */
76
- addJob(job) {
77
- if (!job.schedule) {
78
- return; // Only scheduled jobs go in the heap
79
- }
80
- const nextRun = this.calculateNextRun(job);
81
- if (nextRun === null) {
82
- return; // No valid schedule
83
- }
84
- const entry = {
85
- jobId: job.id,
86
- jobName: job.name,
87
- nextRun,
88
- job,
89
- };
90
- // Remove existing entry if present
91
- if (this.jobMap.has(job.id)) {
92
- this.heap.removeById(job.id);
93
- }
94
- this.heap.push(entry);
95
- this.jobMap.set(job.id, entry);
96
- this.metrics.totalJobs = this.jobMap.size;
97
- if (this.config.debug) {
98
- this.logger.debug(`Added job ${job.id} (${job.name}), next run: ${new Date(nextRun).toISOString()}`);
99
- }
100
- // Reschedule check if this job is due sooner than the current next check
101
- if (this.isRunning && nextRun < this.metrics.nextCheckTime) {
102
- this.scheduleNextCheck();
103
- }
104
- }
105
- /**
106
- * Remove a job from the scheduler
107
- * @param jobId The job ID to remove
108
- * @returns true if removed, false if not found
109
- */
110
- removeJob(jobId) {
111
- const removed = this.heap.removeById(jobId);
112
- const existed = this.jobMap.delete(jobId);
113
- this.lastRunTimes.delete(jobId);
114
- this.metrics.totalJobs = this.jobMap.size;
115
- return existed || removed !== undefined;
116
- }
117
- /**
118
- * Update a job in the scheduler
119
- * @param job Updated job specification
120
- */
121
- updateJob(job) {
122
- this.removeJob(job.id);
123
- this.addJob(job);
124
- }
125
- /**
126
- * Get jobs that are currently due
127
- * @returns Array of jobs ready to execute
128
- */
129
- getDueJobs() {
130
- const startTime = Date.now();
131
- const now = startTime + this.config.dueBuffer;
132
- const dueJobs = [];
133
- // Pop all jobs that are due
134
- while (!this.heap.isEmpty() && this.heap.peek().nextRun <= now) {
135
- const entry = this.heap.pop();
136
- const job = entry.job;
137
- // Check if we haven't run this job in the current minute (for cron jobs)
138
- if (job.schedule?.cron) {
139
- const currentMinute = Math.floor(now / 60000);
140
- const lastRun = this.lastRunTimes.get(job.id);
141
- if (lastRun === currentMinute) {
142
- // Already ran this minute, reschedule for next run (force recalculate)
143
- const nextRun = this.calculateNextRun(job, new Date(now + 60000), true);
144
- if (nextRun !== null) {
145
- entry.nextRun = nextRun;
146
- this.heap.push(entry);
147
- }
148
- continue;
149
- }
150
- this.lastRunTimes.set(job.id, currentMinute);
151
- }
152
- dueJobs.push(job);
153
- // Reschedule for next run if it's a recurring job (force recalculate)
154
- const nextRun = this.calculateNextRun(job, new Date(now), true);
155
- if (nextRun !== null) {
156
- entry.nextRun = nextRun;
157
- entry.job = job; // Keep updated job reference
158
- this.heap.push(entry);
159
- this.jobMap.set(job.id, entry);
160
- }
161
- else {
162
- this.jobMap.delete(job.id);
163
- }
164
- }
165
- // Update metrics
166
- const checkTime = Date.now() - startTime;
167
- this.recordCheckTiming(checkTime);
168
- this.metrics.dueJobs = dueJobs.length;
169
- this.metrics.totalExecuted += dueJobs.length;
170
- return dueJobs;
171
- }
172
- /**
173
- * Check and execute due jobs
174
- * Emits 'jobDue' event for each job that should be executed
175
- */
176
- checkScheduledJobs() {
177
- this.metrics.totalChecks++;
178
- const dueJobs = this.getDueJobs();
179
- for (const job of dueJobs) {
180
- this.emit('jobDue', job);
181
- }
182
- // Schedule next check
183
- if (this.isRunning) {
184
- this.scheduleNextCheck();
185
- }
186
- return dueJobs;
187
- }
188
- /**
189
- * Get scheduler metrics
190
- */
191
- getMetrics() {
192
- return {
193
- ...this.metrics,
194
- totalJobs: this.jobMap.size,
195
- memoryUsage: this.estimateMemoryUsage(),
196
- };
197
- }
198
- /**
199
- * Calculate the next run time for a job
200
- * @param job The job specification
201
- * @param fromDate Optional date to calculate from (used for rescheduling after due)
202
- * @param forceRecalculate If true, ignore existing nextRun and calculate fresh
203
- */
204
- calculateNextRun(job, fromDate, forceRecalculate = false) {
205
- const now = fromDate || new Date();
206
- if (job.schedule?.cron) {
207
- return this.getNextCronRun(job.schedule.cron, now);
208
- }
209
- if (job.schedule?.interval) {
210
- // When adding a new job, use the provided nextRun (even if in the past - job is due)
211
- if (job.schedule.nextRun && !forceRecalculate) {
212
- const nextRunTime = job.schedule.nextRun instanceof Date
213
- ? job.schedule.nextRun.getTime()
214
- : new Date(job.schedule.nextRun).getTime();
215
- return nextRunTime;
216
- }
217
- // Calculate next interval run (used when rescheduling after execution)
218
- return now.getTime() + job.schedule.interval;
219
- }
220
- return null;
221
- }
222
- /**
223
- * Calculate next cron run time
224
- */
225
- getNextCronRun(cronExpr, from) {
226
- try {
227
- const [minute, hour, day, month, weekday] = cronExpr.split(' ');
228
- const now = new Date(from);
229
- // Try to find next matching time within the next 32 days
230
- // (covers monthly cron expressions like "0 0 1 * *")
231
- for (let i = 0; i < 32 * 24 * 60; i++) {
232
- const checkTime = new Date(now.getTime() + i * 60000);
233
- if (this.matchesCronField(minute, checkTime.getMinutes(), 0, 59) &&
234
- this.matchesCronField(hour, checkTime.getHours(), 0, 23) &&
235
- this.matchesCronField(day, checkTime.getDate(), 1, 31) &&
236
- this.matchesCronField(month, checkTime.getMonth() + 1, 1, 12) &&
237
- this.matchesCronField(weekday, checkTime.getDay(), 0, 6)) {
238
- // Round to start of minute
239
- checkTime.setSeconds(0, 0);
240
- // Only return if it's in the future
241
- if (checkTime.getTime() > from.getTime()) {
242
- return checkTime.getTime();
243
- }
244
- }
245
- }
246
- return null;
247
- }
248
- catch (_error) {
249
- this.logger.error(`Invalid cron expression: ${cronExpr}`);
250
- return null;
251
- }
252
- }
253
- /**
254
- * Match a single cron field against a value
255
- */
256
- matchesCronField(field, value, _min, _max) {
257
- // Handle wildcard
258
- if (field === '*') {
259
- return true;
260
- }
261
- // Handle specific number
262
- if (/^\d+$/.test(field)) {
263
- return parseInt(field, 10) === value;
264
- }
265
- // Handle intervals (*/5)
266
- if (field.startsWith('*/')) {
267
- const interval = parseInt(field.substring(2), 10);
268
- return value % interval === 0;
269
- }
270
- // Handle ranges (1-5)
271
- if (field.includes('-') && !field.includes(',')) {
272
- const [start, end] = field.split('-').map(x => parseInt(x, 10));
273
- return value >= start && value <= end;
274
- }
275
- // Handle lists (1,3,5)
276
- if (field.includes(',')) {
277
- const values = field.split(',').map(x => parseInt(x.trim(), 10));
278
- return values.includes(value);
279
- }
280
- // Handle step values (1-10/2)
281
- if (field.includes('/')) {
282
- const [range, step] = field.split('/');
283
- const stepNum = parseInt(step, 10);
284
- if (range === '*') {
285
- return value % stepNum === 0;
286
- }
287
- if (range.includes('-')) {
288
- const [start, end] = range.split('-').map(x => parseInt(x, 10));
289
- if (value < start || value > end)
290
- return false;
291
- return (value - start) % stepNum === 0;
292
- }
293
- }
294
- return false;
295
- }
296
- /**
297
- * Schedule the next check based on when jobs are due
298
- */
299
- scheduleNextCheck() {
300
- if (this.checkTimer) {
301
- clearTimeout(this.checkTimer);
302
- }
303
- if (!this.isRunning) {
304
- return;
305
- }
306
- let delay;
307
- if (this.heap.isEmpty()) {
308
- // No jobs, wait for max interval
309
- delay = this.config.maxCheckInterval;
310
- }
311
- else {
312
- const nextJob = this.heap.peek();
313
- const timeUntilDue = nextJob.nextRun - Date.now();
314
- // Clamp delay between min and max
315
- delay = Math.max(this.config.minCheckInterval, Math.min(this.config.maxCheckInterval, timeUntilDue));
316
- }
317
- this.metrics.nextCheckTime = Date.now() + delay;
318
- this.checkTimer = setTimeout(() => {
319
- this.checkScheduledJobs();
320
- }, delay);
321
- }
322
- /**
323
- * Record check timing for metrics
324
- */
325
- recordCheckTiming(timing) {
326
- this.checkTimings.push(timing);
327
- if (this.checkTimings.length > this.MAX_TIMING_SAMPLES) {
328
- this.checkTimings.shift();
329
- }
330
- this.metrics.averageCheckTime =
331
- this.checkTimings.reduce((sum, t) => sum + t, 0) / this.checkTimings.length;
332
- }
333
- /**
334
- * Estimate memory usage
335
- */
336
- estimateMemoryUsage() {
337
- // Rough estimate: ~500 bytes per entry
338
- return this.jobMap.size * 500;
339
- }
340
- /**
341
- * Get time until next job is due
342
- */
343
- getTimeUntilNextJob() {
344
- if (this.heap.isEmpty()) {
345
- return null;
346
- }
347
- return Math.max(0, this.heap.peek().nextRun - Date.now());
348
- }
349
- /**
350
- * Get all scheduled jobs (for debugging/status)
351
- */
352
- getAllJobs() {
353
- return Array.from(this.jobMap.values());
354
- }
355
- }
356
- export default OptimizedJobScheduler;
@@ -1,215 +0,0 @@
1
- /**
2
- * LSH SaaS Audit Logging Service
3
- * Comprehensive audit trail for all actions
4
- */
5
- import { getSupabaseClient } from './supabase-client.js';
6
- /**
7
- * Audit Logger Service
8
- */
9
- export class AuditLogger {
10
- supabase = getSupabaseClient();
11
- /**
12
- * Log an audit event
13
- */
14
- async log(input) {
15
- try {
16
- const { error } = await this.supabase.from('audit_logs').insert({
17
- organization_id: input.organizationId,
18
- team_id: input.teamId || null,
19
- user_id: input.userId || null,
20
- user_email: input.userEmail || null,
21
- action: input.action,
22
- resource_type: input.resourceType,
23
- resource_id: input.resourceId || null,
24
- ip_address: input.ipAddress || null,
25
- user_agent: input.userAgent || null,
26
- metadata: input.metadata || {},
27
- old_value: input.oldValue || null,
28
- new_value: input.newValue || null,
29
- timestamp: new Date().toISOString(),
30
- });
31
- if (error) {
32
- console.error('Failed to write audit log:', error);
33
- // Don't throw - audit logging should not break the main operation
34
- }
35
- }
36
- catch (error) {
37
- console.error('Audit logging error:', error);
38
- // Don't throw - audit logging should not break the main operation
39
- }
40
- }
41
- /**
42
- * Get audit logs for organization
43
- */
44
- async getOrganizationLogs(organizationId, options = {}) {
45
- let query = this.supabase
46
- .from('audit_logs')
47
- .select('*', { count: 'exact' })
48
- .eq('organization_id', organizationId);
49
- if (options.startDate) {
50
- query = query.gte('timestamp', options.startDate.toISOString());
51
- }
52
- if (options.endDate) {
53
- query = query.lte('timestamp', options.endDate.toISOString());
54
- }
55
- if (options.action) {
56
- query = query.eq('action', options.action);
57
- }
58
- if (options.userId) {
59
- query = query.eq('user_id', options.userId);
60
- }
61
- if (options.teamId) {
62
- query = query.eq('team_id', options.teamId);
63
- }
64
- query = query.order('timestamp', { ascending: false });
65
- if (options.limit) {
66
- query = query.limit(options.limit);
67
- }
68
- if (options.offset) {
69
- query = query.range(options.offset, options.offset + (options.limit || 50) - 1);
70
- }
71
- const { data, count, error } = await query;
72
- if (error) {
73
- throw new Error(`Failed to get audit logs: ${error.message}`);
74
- }
75
- return {
76
- logs: (data || []).map(this.mapDbLogToLog),
77
- total: count || 0,
78
- };
79
- }
80
- /**
81
- * Get audit logs for a specific resource
82
- */
83
- async getResourceLogs(organizationId, resourceType, resourceId, limit = 50) {
84
- const { data, error } = await this.supabase
85
- .from('audit_logs')
86
- .select('*')
87
- .eq('organization_id', organizationId)
88
- .eq('resource_type', resourceType)
89
- .eq('resource_id', resourceId)
90
- .order('timestamp', { ascending: false })
91
- .limit(limit);
92
- if (error) {
93
- throw new Error(`Failed to get resource logs: ${error.message}`);
94
- }
95
- return (data || []).map(this.mapDbLogToLog);
96
- }
97
- /**
98
- * Get audit logs for a team
99
- */
100
- async getTeamLogs(teamId, options = {}) {
101
- let query = this.supabase
102
- .from('audit_logs')
103
- .select('*', { count: 'exact' })
104
- .eq('team_id', teamId);
105
- if (options.startDate) {
106
- query = query.gte('timestamp', options.startDate.toISOString());
107
- }
108
- if (options.endDate) {
109
- query = query.lte('timestamp', options.endDate.toISOString());
110
- }
111
- query = query.order('timestamp', { ascending: false });
112
- if (options.limit) {
113
- query = query.limit(options.limit);
114
- }
115
- if (options.offset) {
116
- query = query.range(options.offset, options.offset + (options.limit || 50) - 1);
117
- }
118
- const { data, count, error } = await query;
119
- if (error) {
120
- throw new Error(`Failed to get team logs: ${error.message}`);
121
- }
122
- return {
123
- logs: (data || []).map(this.mapDbLogToLog),
124
- total: count || 0,
125
- };
126
- }
127
- /**
128
- * Get audit logs for a user
129
- */
130
- async getUserLogs(userId, options = {}) {
131
- let query = this.supabase
132
- .from('audit_logs')
133
- .select('*', { count: 'exact' })
134
- .eq('user_id', userId);
135
- if (options.startDate) {
136
- query = query.gte('timestamp', options.startDate.toISOString());
137
- }
138
- if (options.endDate) {
139
- query = query.lte('timestamp', options.endDate.toISOString());
140
- }
141
- query = query.order('timestamp', { ascending: false });
142
- if (options.limit) {
143
- query = query.limit(options.limit);
144
- }
145
- if (options.offset) {
146
- query = query.range(options.offset, options.offset + (options.limit || 50) - 1);
147
- }
148
- const { data, count, error } = await query;
149
- if (error) {
150
- throw new Error(`Failed to get user logs: ${error.message}`);
151
- }
152
- return {
153
- logs: (data || []).map(this.mapDbLogToLog),
154
- total: count || 0,
155
- };
156
- }
157
- /**
158
- * Delete old audit logs (for retention policy)
159
- */
160
- async deleteOldLogs(organizationId, retentionDays) {
161
- const cutoffDate = new Date();
162
- cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
163
- const { count, error } = await this.supabase
164
- .from('audit_logs')
165
- .delete({ count: 'exact' })
166
- .eq('organization_id', organizationId)
167
- .lt('timestamp', cutoffDate.toISOString());
168
- if (error) {
169
- throw new Error(`Failed to delete old logs: ${error.message}`);
170
- }
171
- return count || 0;
172
- }
173
- /**
174
- * Map database log to AuditLog type
175
- */
176
- mapDbLogToLog(dbLog) {
177
- return {
178
- id: dbLog.id,
179
- organizationId: dbLog.organization_id,
180
- teamId: dbLog.team_id,
181
- userId: dbLog.user_id,
182
- userEmail: dbLog.user_email,
183
- action: dbLog.action,
184
- resourceType: dbLog.resource_type,
185
- resourceId: dbLog.resource_id,
186
- ipAddress: dbLog.ip_address,
187
- userAgent: dbLog.user_agent,
188
- metadata: dbLog.metadata || {},
189
- oldValue: dbLog.old_value,
190
- newValue: dbLog.new_value,
191
- timestamp: new Date(dbLog.timestamp),
192
- };
193
- }
194
- }
195
- /**
196
- * Singleton instance
197
- */
198
- export const auditLogger = new AuditLogger();
199
- /**
200
- * Helper function to extract IP from request
201
- */
202
- export function getIpFromRequest(req) {
203
- const forwarded = req.headers['x-forwarded-for'];
204
- const forwardedIp = typeof forwarded === 'string' ? forwarded.split(',')[0].trim() : undefined;
205
- const realIp = req.headers['x-real-ip'];
206
- return (forwardedIp ||
207
- (typeof realIp === 'string' ? realIp : undefined) ||
208
- req.socket?.remoteAddress);
209
- }
210
- /**
211
- * Helper function to extract user agent from request
212
- */
213
- export function getUserAgentFromRequest(req) {
214
- return req.headers['user-agent'];
215
- }