holosphere 1.1.20 → 2.0.0-alpha1

package/src/subscriptions/manager.js

@@ -0,0 +1,142 @@
+/**
+ * Real-time Subscription Management
+ */
+
+import { subscribe } from '../storage/nostr-wrapper.js';
+import { resolveHologram } from '../federation/hologram.js';
+
+/**
+ * Create subscription with throttle and filter
+ * @param {Object} client - Nostr client instance
+ * @param {string} path - Path to subscribe to
+ * @param {Function} callback - Callback function
+ * @param {Object} options - Subscription options
+ * @param {number} options.throttle - Throttle interval in ms
+ * @param {Function} options.filter - Filter predicate
+ * @param {boolean} options.includeFederated - Include federated data
+ * @param {boolean} options.triggerInitial - Trigger callback with initial data (default: false)
+ * @param {boolean} options.resolveHolograms - Resolve holograms before callback (default: true)
+ * @returns {Object} Subscription object with unsubscribe method
+ */
+export async function createSubscription(client, path, callback, options = {}) {
+  const { throttle = 0, filter = null, includeFederated = false, triggerInitial = false, realtimeOnly = true, resolveHolograms = true } = options;
+
+  let lastInvoke = 0;
+  let timeoutId = null;
+  const seenKeys = new Set();
+  let isActive = true;  // Track active state
+
+  const wrappedCallback = async (data, key) => {
+    // Check if subscription is still active
+    if (!isActive) {
+      return;
+    }
+
+    // Skip if already seen (prevent duplicates)
+    // Include timestamp in unique key to allow updates to same item
+    const uniqueKey = `${key}-${data?.id || ''}-${data?._meta?.timestamp || Date.now()}`;
+    if (seenKeys.has(uniqueKey)) {
+      return;
+    }
+    seenKeys.add(uniqueKey);
+
+    // Resolve holograms if enabled and data is a hologram
+    let resolvedData = data;
+    if (resolveHolograms && data && data.hologram === true) {
+      try {
+        resolvedData = await resolveHologram(client, data);
+        // If resolution failed, skip this item (source data may not exist yet)
+        if (!resolvedData) {
+          return;
+        }
+      } catch (err) {
+        console.warn('Failed to resolve hologram in subscription:', err);
+        return;
+      }
+    }
+
+    // Apply filter (on resolved data)
+    if (filter && !filter(resolvedData, key)) {
+      return;
+    }
+
+    // Apply throttle
+    const now = Date.now();
+    if (throttle > 0) {
+      if (now - lastInvoke < throttle) {
+        // Clear existing timeout
+        if (timeoutId) clearTimeout(timeoutId);
+
+        // Schedule callback
+        timeoutId = setTimeout(() => {
+          lastInvoke = Date.now();
+          callback(resolvedData, key);
+        }, throttle - (now - lastInvoke));
+        return;
+      }
+    }
+
+    lastInvoke = now;
+    callback(resolvedData, key);
+  };
+
+  // Subscribe using Nostr wrapper
+  const subscription = await subscribe(client, path, wrappedCallback, { realtimeOnly });
+
+  // Return subscription object
+  return {
+    path,
+    unsubscribe: () => {
+      isActive = false;  // Mark as inactive immediately
+      if (timeoutId) clearTimeout(timeoutId);
+      if (subscription && subscription.unsubscribe) {
+        subscription.unsubscribe();
+      }
+      seenKeys.clear();
+    },
+  };
+}
+
+/**
+ * Subscription registry for tracking active subscriptions
+ */
+export class SubscriptionRegistry {
+  constructor() {
+    this.subscriptions = new Map();
+  }
+
+  /**
+   * Register a subscription
+   */
+  register(id, subscription) {
+    this.subscriptions.set(id, subscription);
+  }
+
+  /**
+   * Unregister a subscription
+   */
+  unregister(id) {
+    const subscription = this.subscriptions.get(id);
+    if (subscription) {
+      subscription.unsubscribe();
+      this.subscriptions.delete(id);
+    }
+  }
+
+  /**
+   * Unsubscribe all
+   */
+  unsubscribeAll() {
+    for (const [id, subscription] of this.subscriptions) {
+      subscription.unsubscribe();
+    }
+    this.subscriptions.clear();
+  }
+
+  /**
+   * Get active subscription count
+   */
+  count() {
+    return this.subscriptions.size;
+  }
+}

package/test-ai-real-api.js

@@ -0,0 +1,202 @@
+/**
+ * Test AI Module with Real OpenAI API Calls
+ *
+ * Run with: node test-ai-real-api.js
+ *
+ * Requires OPENAI_API_KEY environment variable to be set
+ */
+
+import 'dotenv/config';
+import { LLMService } from './src/ai/llm-service.js';
+import { Embeddings } from './src/ai/embeddings.js';
+import { SchemaExtractor } from './src/ai/schema-extractor.js';
+import { Classifier } from './src/ai/classifier.js';
+import { NLQuery } from './src/ai/nl-query.js';
+import { createAIServices } from './src/ai/index.js';
+import OpenAI from 'openai';
+
+const apiKey = process.env.OPENAI_API_KEY;
+
+if (!apiKey) {
+  console.error('ERROR: OPENAI_API_KEY environment variable is not set');
+  console.error('Please set it in your .env file or export it:');
+  console.error('  export OPENAI_API_KEY=sk-...');
+  process.exit(1);
+}
+
+console.log('='.repeat(60));
+console.log('  HoloSphere AI Module - Real API Test');
+console.log('='.repeat(60));
+console.log(`API Key: ${apiKey.substring(0, 10)}...${apiKey.substring(apiKey.length - 4)}`);
+console.log('');
+
+let passed = 0;
+let failed = 0;
+
+async function test(name, fn) {
+  process.stdout.write(`Testing ${name}... `);
+  try {
+    const result = await fn();
+    console.log('PASSED');
+    if (result) console.log(`  Result: ${JSON.stringify(result).substring(0, 100)}...`);
+    passed++;
+    return result;
+  } catch (err) {
+    console.log('FAILED');
+    console.log(`  Error: ${err.message}`);
+    failed++;
+    return null;
+  }
+}
+
+// Initialize services
+const llm = new LLMService(apiKey);
+const openai = new OpenAI({ apiKey });
+const embeddings = new Embeddings(openai);
+
+console.log('\n--- LLM Service Tests ---\n');
+
+// Test 1: Basic chat completion
+await test('LLM sendMessage', async () => {
+  const response = await llm.sendMessage(
+    'You are a helpful assistant. Respond in one sentence.',
+    'What is H3 hexagonal indexing?'
+  );
+  return response.substring(0, 80);
+});
+
+// Test 2: Summarization
+await test('LLM summarize', async () => {
+  const text = `
+    HoloSphere is a local-first, geospatially-aware database built on Nostr.
+    It uses H3 hexagonal cells to partition geographic space into holons.
+    Each holon can store data in different lenses for domain separation.
+    Federation allows holons to share data through lightweight references called holograms.
+  `;
+  const summary = await llm.summarize(text, { maxLength: 30 });
+  return summary;
+});
+
+// Test 3: Keyword extraction
+await test('LLM extractKeywords', async () => {
+  const keywords = await llm.extractKeywords(
+    'HoloSphere uses H3 hexagonal cells and Nostr protocol for geospatial data storage',
+    5
+  );
+  return keywords;
+});
+
+// Test 4: JSON extraction
+await test('LLM getJSON', async () => {
+  const json = await llm.getJSON(
+    'Extract the person info as JSON with fields: name, age, city',
+    'John Smith is 32 years old and lives in San Francisco'
+  );
+  return json;
+});
+
+console.log('\n--- Embeddings Tests ---\n');
+
+// Test 5: Single embedding
+await test('Embeddings embed', async () => {
+  const embedding = await embeddings.embed('HoloSphere geospatial database');
+  return { dimensions: embedding.length, sample: embedding.slice(0, 3) };
+});
+
+// Test 6: Batch embeddings
+await test('Embeddings embedBatch', async () => {
+  const batch = await embeddings.embedBatch([
+    'Geographic data',
+    'Hexagonal cells',
+    'Federation'
+  ]);
+  return { count: batch.length, dimensions: batch[0]?.length };
+});
+
+// Test 7: Similarity calculation
+await test('Embeddings similarity', async () => {
+  const emb1 = await embeddings.embed('San Francisco California');
+  const emb2 = await embeddings.embed('Los Angeles California');
+  const emb3 = await embeddings.embed('Programming code software');
+
+  const sim12 = embeddings.cosineSimilarity(emb1, emb2);
+  const sim13 = embeddings.cosineSimilarity(emb1, emb3);
+
+  return {
+    'SF vs LA': sim12.toFixed(3),
+    'SF vs Code': sim13.toFixed(3),
+    'Cities more similar': sim12 > sim13
+  };
+});
+
+console.log('\n--- Schema Extractor Tests ---\n');
+
+// Test 8: Schema extraction
+await test('SchemaExtractor extractToSchema', async () => {
+  const extractor = new SchemaExtractor(llm);
+  const schema = {
+    type: 'object',
+    properties: {
+      location: { type: 'string', description: 'Location name' },
+      latitude: { type: 'number', description: 'Latitude coordinate' },
+      longitude: { type: 'number', description: 'Longitude coordinate' }
+    },
+    required: ['location', 'latitude', 'longitude']
+  };
+
+  const data = await extractor.extractToSchema(
+    'The Eiffel Tower is located in Paris at coordinates 48.8584 N, 2.2945 E',
+    schema
+  );
+  return data;
+});
+
+console.log('\n--- Classifier Tests ---\n');
+
+// Test 9: Content classification
+await test('Classifier classifyToLens', async () => {
+  const classifier = new Classifier(llm);
+  classifier.registerLenses({
+    events: { description: 'Events, meetings, gatherings' },
+    projects: { description: 'Projects, initiatives, work items' },
+    resources: { description: 'Resources, documents, materials' },
+    places: { description: 'Locations, venues, geographic points' }
+  });
+
+  const result = await classifier.classifyToLens({
+    title: 'Community Garden Project',
+    description: 'A local initiative to create a shared vegetable garden in the park'
+  });
+  return result;
+});
+
+console.log('\n--- NL Query Tests ---\n');
+
+// Test 10: Natural language to query
+await test('NLQuery parse', async () => {
+  const nlQuery = new NLQuery(llm);
+  const query = await nlQuery.parse(
+    'Find all coffee shops within 500 meters of downtown San Francisco'
+  );
+  return query;
+});
+
+console.log('\n--- Full Service Creation ---\n');
+
+// Test 11: Create all services
+await test('createAIServices', async () => {
+  const services = createAIServices(apiKey, null, {}, openai);
+  return {
+    servicesCreated: Object.keys(services).length,
+    services: Object.keys(services)
+  };
+});
+
+// Summary
+console.log('\n' + '='.repeat(60));
+console.log(`  Results: ${passed} passed, ${failed} failed`);
+console.log('='.repeat(60));
+
+if (failed > 0) {
+  process.exit(1);
+}

package/tests/unit/ai/aggregation.test.js

@@ -0,0 +1,295 @@
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { SmartAggregation } from '../../../src/ai/aggregation.js';
+
+// Mock h3-js
+vi.mock('h3-js', () => ({
+  getResolution: vi.fn().mockReturnValue(5),
+  cellToParent: vi.fn().mockReturnValue('parent_holon'),
+  cellToChildren: vi.fn().mockReturnValue(['child1', 'child2', 'child3'])
+}));
+
+describe('Unit: SmartAggregation', () => {
+  let aggregation;
+  let mockLLM;
+  let mockHolosphere;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+
+    mockLLM = {
+      getJSON: vi.fn()
+    };
+
+    mockHolosphere = {
+      getAll: vi.fn().mockResolvedValue([]),
+      put: vi.fn().mockResolvedValue({ success: true })
+    };
+
+    aggregation = new SmartAggregation(mockLLM, mockHolosphere);
+  });
+
+  describe('Constructor', () => {
+    it('should initialize with LLM service', () => {
+      const a = new SmartAggregation(mockLLM);
+      expect(a.llm).toBe(mockLLM);
+      expect(a.holosphere).toBeNull();
+    });
+
+    it('should accept optional HoloSphere instance', () => {
+      expect(aggregation.holosphere).toBe(mockHolosphere);
+    });
+  });
+
+  describe('setHoloSphere', () => {
+    it('should set HoloSphere instance', () => {
+      const a = new SmartAggregation(mockLLM);
+      a.setHoloSphere(mockHolosphere);
+      expect(a.holosphere).toBe(mockHolosphere);
+    });
+  });
+
+  describe('smartUpcast', () => {
+    it('should generate summary and upcast to parent', async () => {
+      mockHolosphere.getAll.mockResolvedValue([
+        { id: 1, title: 'Item 1' },
+        { id: 2, title: 'Item 2' }
+      ]);
+
+      mockLLM.getJSON.mockResolvedValue({
+        text: 'Two items in region',
+        stats: { count: 2 },
+        notable: ['Item 1'],
+        themes: ['development']
+      });
+
+      const result = await aggregation.smartUpcast('holon1', 'projects');
+
+      expect(result.holon).toBe('holon1');
+      expect(result.lens).toBe('projects');
+      expect(result.summary).toBeDefined();
+      expect(result.dataCount).toBe(2);
+      expect(result.parent).toBeDefined();
+    });
+
+    it('should throw error if HoloSphere not available', async () => {
+      const a = new SmartAggregation(mockLLM);
+
+      await expect(a.smartUpcast('holon', 'lens'))
+        .rejects.toThrow('HoloSphere instance required');
+    });
+
+    it('should store summary at parent level when storeResults is true', async () => {
+      mockHolosphere.getAll.mockResolvedValue([{ id: 1 }]);
+      mockLLM.getJSON.mockResolvedValue({ text: 'Summary' });
+
+      await aggregation.smartUpcast('holon1', 'projects', { storeResults: true });
+
+      expect(mockHolosphere.put).toHaveBeenCalledWith(
+        'parent_holon',
+        'projects_summaries',
+        expect.objectContaining({
+          childHolon: 'holon1',
+          summary: expect.any(Object)
+        })
+      );
+    });
+
+    it('should respect maxLevels option', async () => {
+      mockHolosphere.getAll.mockResolvedValue([{ id: 1 }]);
+      mockLLM.getJSON.mockResolvedValue({ text: 'Summary' });
+
+      const result = await aggregation.smartUpcast('holon1', 'projects', { maxLevels: 1 });
+
+      // Should only go up one level
+      expect(result.parent).toBeDefined();
+    });
+  });
+
+  describe('_generateSummary', () => {
+    it('should generate summary for data', async () => {
+      mockLLM.getJSON.mockResolvedValue({
+        text: 'Active region',
+        stats: { count: 5 },
+        notable: ['Project A'],
+        themes: ['sustainability']
+      });
+
+      const data = [{ id: 1 }, { id: 2 }];
+      const result = await aggregation._generateSummary(data, 'holon1', 'projects');
+
+      expect(result.text).toBe('Active region');
+      expect(result.stats.count).toBe(5);
+    });
+
+    it('should return empty summary for empty data', async () => {
+      const result = await aggregation._generateSummary([], 'holon1', 'lens');
+
+      expect(result.text).toBe('No data in this region');
+      expect(result.stats.count).toBe(0);
+    });
+  });
+
+  describe('generateHolonSummary', () => {
+    it('should generate comprehensive holon summary', async () => {
+      mockHolosphere.getAll
+        .mockResolvedValueOnce([{ id: 1 }])  // projects
+        .mockResolvedValueOnce([{ id: 2 }])  // quests
+        .mockResolvedValueOnce([])           // events
+        .mockResolvedValueOnce([])           // resources
+        .mockResolvedValueOnce([{ id: 3 }]); // default
+
+      mockLLM.getJSON.mockResolvedValue({
+        title: 'Region Summary',
+        executive_summary: 'Active region',
+        highlights: ['Growth'],
+        activities: { count: 2, summary: 'Various activities' },
+        community: { engagement_level: 'high', notes: 'Active community' },
+        resources: ['Local resources'],
+        challenges: ['Coordination'],
+        opportunities: ['Expansion'],
+        health_score: 0.8,
+        recommendations: ['Focus on coordination']
+      });
+
+      const result = await aggregation.generateHolonSummary('holon1');
+
+      expect(result.holon).toBe('holon1');
+      expect(result.summary.title).toBe('Region Summary');
+      expect(result.lensesAnalyzed).toBeDefined();
+      expect(result.totalItems).toBeGreaterThan(0);
+    });
+
+    it('should throw error if HoloSphere not available', async () => {
+      const a = new SmartAggregation(mockLLM);
+
+      await expect(a.generateHolonSummary('holon'))
+        .rejects.toThrow('HoloSphere instance required');
+    });
+
+    it('should skip unavailable lenses', async () => {
+      mockHolosphere.getAll
+        .mockResolvedValueOnce([{ id: 1 }])
+        .mockRejectedValueOnce(new Error('Lens not found'))
+        .mockResolvedValueOnce([])
+        .mockResolvedValueOnce([])
+        .mockResolvedValueOnce([]);
+
+      mockLLM.getJSON.mockResolvedValue({ title: 'Summary' });
+
+      const result = await aggregation.generateHolonSummary('holon1');
+
+      expect(result).toBeDefined();
+    });
+  });
+
+  describe('aggregateChildren', () => {
+    it('should aggregate summaries from child holons', async () => {
+      mockHolosphere.getAll
+        .mockResolvedValueOnce([{ id: 1 }])
+        .mockResolvedValueOnce([{ id: 2 }])
+        .mockResolvedValueOnce([{ id: 3 }]);
+
+      mockLLM.getJSON.mockResolvedValue({
+        text: 'Summary',
+        stats: { count: 1 }
+      });
+
+      const result = await aggregation.aggregateChildren('parent_holon', 'projects');
+
+      expect(result.holon).toBe('parent_holon');
+      expect(result.lens).toBe('projects');
+      expect(result.childCount).toBeGreaterThan(0);
+    });
+
+    it('should throw error if HoloSphere not available', async () => {
+      const a = new SmartAggregation(mockLLM);
+
+      await expect(a.aggregateChildren('parent', 'lens'))
+        .rejects.toThrow('HoloSphere instance required');
+    });
+
+    it('should return message when no child data found', async () => {
+      mockHolosphere.getAll.mockResolvedValue([]);
+
+      const result = await aggregation.aggregateChildren('parent_holon', 'projects');
+
+      expect(result.message).toBe('No child data found');
+      expect(result.summary).toBeNull();
+    });
+  });
+
+  describe('comparePeriods', () => {
+    it('should compare data across time periods', async () => {
+      mockHolosphere.getAll.mockResolvedValue([
+        { id: 1, timestamp: '2024-01-15' },
+        { id: 2, timestamp: '2024-06-15' },
+        { id: 3, timestamp: '2024-08-15' }
+      ]);
+
+      mockLLM.getJSON.mockResolvedValue({
+        growth_rate: 50,
+        direction: 'growth',
+        new_themes: ['sustainability'],
+        disappeared: ['legacy'],
+        key_changes: ['Focus shift'],
+        summary: 'Growing activity'
+      });
+
+      const result = await aggregation.comparePeriods(
+        'holon1',
+        'projects',
+        { start: '2024-01-01', end: '2024-03-31' },
+        { start: '2024-07-01', end: '2024-09-30' }
+      );
+
+      expect(result.growth_rate).toBe(50);
+      expect(result.direction).toBe('growth');
+    });
+
+    it('should throw error if HoloSphere not available', async () => {
+      const a = new SmartAggregation(mockLLM);
+
+      await expect(a.comparePeriods('h', 'l', {}, {}))
+        .rejects.toThrow('HoloSphere instance required');
+    });
+
+    it('should filter items by period', async () => {
+      mockHolosphere.getAll.mockResolvedValue([
+        { id: 1, timestamp: '2024-01-15' },
+        { id: 2, timestamp: '2024-02-15' },
+        { id: 3, timestamp: '2024-06-15' }
+      ]);
+
+      mockLLM.getJSON.mockResolvedValue({ growth_rate: 0 });
+
+      await aggregation.comparePeriods(
+        'holon1',
+        'lens',
+        { start: '2024-01-01', end: '2024-03-31' },
+        { start: '2024-06-01', end: '2024-06-30' }
+      );
+
+      const call = mockLLM.getJSON.mock.calls[0];
+      expect(call[1]).toContain('Period 1 data');
+      expect(call[1]).toContain('Period 2 data');
+    });
+
+    it('should handle items with created_at field', async () => {
+      mockHolosphere.getAll.mockResolvedValue([
+        { id: 1, created_at: '2024-01-15' },
+        { id: 2, created_at: '2024-06-15' }
+      ]);
+
+      mockLLM.getJSON.mockResolvedValue({ growth_rate: 100 });
+
+      await aggregation.comparePeriods(
+        'holon1',
+        'lens',
+        { start: '2024-01-01', end: '2024-03-31' },
+        { start: '2024-06-01', end: '2024-06-30' }
+      );
+
+      expect(mockLLM.getJSON).toHaveBeenCalled();
+    });
+  });
+});