holosphere 1.1.19 → 2.0.0-alpha0

package/src/ai/schema-extractor.js

@@ -0,0 +1,218 @@
+/**
+ * Schema Extractor - Extract structured data from text using JSON schemas
+ */
+
+/**
+ * Schema Extractor class
+ */
+export class SchemaExtractor {
+  /**
+   * @param {LLMService} llmService - LLM service instance
+   */
+  constructor(llmService) {
+    this.llm = llmService;
+  }
+
+  /**
+   * Extract data from text based on JSON schema
+   * @param {string} text - Text to extract from
+   * @param {Object} schema - JSON schema defining expected structure
+   * @param {Object} options - Extraction options
+   * @returns {Promise<Object>} Extracted data conforming to schema
+   */
+  async extractToSchema(text, schema, options = {}) {
+    const schemaDescription = this._describeSchema(schema);
+
+    const systemPrompt = `You are a data extraction expert. Extract structured information from the provided text according to this JSON schema:
+
+${JSON.stringify(schema, null, 2)}
+
+Schema field descriptions:
+${schemaDescription}
+
+Rules:
+1. Extract ONLY information present in the text
+2. Use null for fields where information is not available
+3. Follow the exact schema structure and field types
+4. For arrays, include all relevant items found
+5. Return ONLY valid JSON matching the schema`;
+
+    const response = await this.llm.getJSON(systemPrompt, text, {
+      temperature: 0.2,
+      maxTokens: options.maxTokens || 2000
+    });
+
+    return response;
+  }
+
+  /**
+   * Generate a human-readable description of the schema
+   * @private
+   */
+  _describeSchema(schema, path = '') {
+    const descriptions = [];
+
+    if (schema.description) {
+      descriptions.push(`${path || 'root'}: ${schema.description}`);
+    }
+
+    if (schema.properties) {
+      for (const [key, prop] of Object.entries(schema.properties)) {
+        const fieldPath = path ? `${path}.${key}` : key;
+        const type = prop.type || 'any';
+        const required = schema.required?.includes(key) ? ' (required)' : '';
+        const desc = prop.description || '';
+
+        descriptions.push(`- ${fieldPath} [${type}]${required}: ${desc}`);
+
+        if (prop.type === 'object' && prop.properties) {
+          descriptions.push(this._describeSchema(prop, fieldPath));
+        }
+
+        if (prop.type === 'array' && prop.items) {
+          if (prop.items.properties) {
+            descriptions.push(this._describeSchema(prop.items, `${fieldPath}[]`));
+          }
+        }
+      }
+    }
+
+    return descriptions.join('\n');
+  }
+
+  /**
+   * Extract multiple items from text
+   * @param {string} text - Text containing multiple items
+   * @param {Object} itemSchema - Schema for each item
+   * @param {Object} options - Options
+   * @returns {Promise<Object[]>} Array of extracted items
+   */
+  async extractMultiple(text, itemSchema, options = {}) {
+    const systemPrompt = `You are a data extraction expert. Extract ALL items matching this schema from the text:
+
+${JSON.stringify(itemSchema, null, 2)}
+
+Return a JSON array of all items found. If no items found, return [].
+Return ONLY valid JSON array.`;
+
+    const response = await this.llm.getJSON(systemPrompt, text, {
+      temperature: 0.2,
+      maxTokens: options.maxTokens || 3000
+    });
+
+    return Array.isArray(response) ? response : [response];
+  }
+
+  /**
+   * Fill missing fields in existing data
+   * @param {Object} data - Existing data with some fields
+   * @param {string} text - Text to extract missing data from
+   * @param {Object} schema - Full schema
+   * @returns {Promise<Object>} Complete data
+   */
+  async fillMissing(data, text, schema) {
+    const missingFields = this._findMissingFields(data, schema);
+
+    if (missingFields.length === 0) {
+      return data;
+    }
+
+    const systemPrompt = `You have existing data:
+${JSON.stringify(data, null, 2)}
+
+Extract ONLY the following missing fields from the text:
+${missingFields.join(', ')}
+
+Based on this schema:
+${JSON.stringify(schema, null, 2)}
+
+Return JSON with ONLY the missing fields that you can extract. Do not modify existing fields.`;
+
+    const extracted = await this.llm.getJSON(systemPrompt, text, { temperature: 0.2 });
+
+    return { ...data, ...extracted };
+  }
+
+  /**
+   * Find fields missing from data based on schema
+   * @private
+   */
+  _findMissingFields(data, schema, path = '') {
+    const missing = [];
+
+    if (schema.properties) {
+      for (const [key, prop] of Object.entries(schema.properties)) {
+        const fieldPath = path ? `${path}.${key}` : key;
+        const value = data[key];
+
+        if (value === undefined || value === null) {
+          missing.push(fieldPath);
+        } else if (prop.type === 'object' && prop.properties) {
+          missing.push(...this._findMissingFields(value, prop, fieldPath));
+        }
+      }
+    }
+
+    return missing;
+  }
+
+  /**
+   * Validate extracted data against schema
+   * @param {Object} data - Extracted data
+   * @param {Object} schema - JSON schema
+   * @returns {{valid: boolean, errors: string[]}}
+   */
+  validateExtraction(data, schema) {
+    const errors = [];
+
+    if (schema.required) {
+      for (const field of schema.required) {
+        if (data[field] === undefined || data[field] === null) {
+          errors.push(`Missing required field: ${field}`);
+        }
+      }
+    }
+
+    if (schema.properties) {
+      for (const [key, prop] of Object.entries(schema.properties)) {
+        const value = data[key];
+
+        if (value !== undefined && value !== null) {
+          const typeValid = this._validateType(value, prop.type);
+          if (!typeValid) {
+            errors.push(`Invalid type for ${key}: expected ${prop.type}, got ${typeof value}`);
+          }
+        }
+      }
+    }
+
+    return {
+      valid: errors.length === 0,
+      errors
+    };
+  }
+
+  /**
+   * Validate value type
+   * @private
+   */
+  _validateType(value, type) {
+    switch (type) {
+      case 'string':
+        return typeof value === 'string';
+      case 'number':
+      case 'integer':
+        return typeof value === 'number';
+      case 'boolean':
+        return typeof value === 'boolean';
+      case 'array':
+        return Array.isArray(value);
+      case 'object':
+        return typeof value === 'object' && !Array.isArray(value);
+      default:
+        return true;
+    }
+  }
+}
+
+export default SchemaExtractor;

package/src/ai/spatial.js

@@ -0,0 +1,293 @@
+/**
+ * Spatial Analysis - AI-powered analysis of data patterns across geographic regions
+ */
+
+/**
+ * Spatial Analysis class
+ */
+export class SpatialAnalysis {
+  /**
+   * @param {LLMService} llmService - LLM service instance
+   * @param {Object} holosphere - HoloSphere instance
+   */
+  constructor(llmService, holosphere = null) {
+    this.llm = llmService;
+    this.holosphere = holosphere;
+  }
+
+  /**
+   * Set HoloSphere instance
+   * @param {Object} holosphere - HoloSphere instance
+   */
+  setHoloSphere(holosphere) {
+    this.holosphere = holosphere;
+  }
+
+  /**
+   * Analyze data patterns in a region
+   * @param {string} holon - Holon identifier
+   * @param {string} lens - Lens to analyze (optional, all if null)
+   * @param {string} aspect - Analysis aspect (activity, trends, distribution, etc.)
+   * @returns {Promise<Object>} Analysis results
+   */
+  async analyzeRegion(holon, lens = null, aspect = 'general') {
+    if (!this.holosphere) {
+      throw new Error('HoloSphere instance required');
+    }
+
+    // Get data from holon
+    let data = [];
+    if (lens) {
+      data = await this.holosphere.getAll(holon, lens);
+    } else {
+      // Get from all known lenses - would need lens list
+      data = await this.holosphere.getAll(holon, 'default');
+    }
+
+    const dataStr = JSON.stringify(data.slice(0, 50), null, 2); // Sample for analysis
+
+    const systemPrompt = `You are a geospatial data analyst. Analyze the following data from a geographic region.
+
+Region (holon): ${holon}
+Lens: ${lens || 'all'}
+Analysis aspect: ${aspect}
+Total items: ${data.length}
+
+Provide insights about:
+1. Key patterns and trends
+2. Notable clusters or concentrations
+3. Anomalies or outliers
+4. Recommendations for the region
+
+Return JSON:
+{
+  "summary": "brief overview",
+  "patterns": ["pattern1", "pattern2"],
+  "insights": ["insight1", "insight2"],
+  "metrics": {"metric1": value},
+  "recommendations": ["rec1", "rec2"],
+  "confidence": 0.0-1.0
+}`;
+
+    const analysis = await this.llm.getJSON(systemPrompt, dataStr, { temperature: 0.3 });
+
+    return {
+      holon,
+      lens,
+      aspect,
+      dataCount: data.length,
+      analysis,
+      timestamp: Date.now(),
+    };
+  }
+
+  /**
+   * Compare data between two regions
+   * @param {string} holon1 - First holon
+   * @param {string} holon2 - Second holon
+   * @param {string} lens - Lens to compare
+   * @returns {Promise<Object>} Comparison results
+   */
+  async compareRegions(holon1, holon2, lens) {
+    if (!this.holosphere) {
+      throw new Error('HoloSphere instance required');
+    }
+
+    const data1 = await this.holosphere.getAll(holon1, lens);
+    const data2 = await this.holosphere.getAll(holon2, lens);
+
+    const systemPrompt = `You are a comparative analyst. Compare data between two geographic regions.
+
+Region 1 (${holon1}): ${data1.length} items
+Region 2 (${holon2}): ${data2.length} items
+Lens: ${lens}
+
+Analyze and compare:
+1. Similarities between regions
+2. Key differences
+3. Relative strengths of each region
+4. Opportunities for collaboration
+
+Return JSON:
+{
+  "summary": "brief comparison",
+  "similarities": ["sim1", "sim2"],
+  "differences": ["diff1", "diff2"],
+  "region1_strengths": ["strength1"],
+  "region2_strengths": ["strength1"],
+  "collaboration_opportunities": ["opp1"],
+  "metrics_comparison": {"metric": {"region1": val, "region2": val}}
+}`;
+
+    const userMessage = `Region 1 data sample:
+${JSON.stringify(data1.slice(0, 25), null, 2)}
+
+Region 2 data sample:
+${JSON.stringify(data2.slice(0, 25), null, 2)}`;
+
+    const comparison = await this.llm.getJSON(systemPrompt, userMessage, { temperature: 0.3 });
+
+    return {
+      regions: [holon1, holon2],
+      lens,
+      dataCounts: { [holon1]: data1.length, [holon2]: data2.length },
+      comparison,
+      timestamp: Date.now(),
+    };
+  }
+
+  /**
+   * Identify spatial trends over time
+   * @param {string} holon - Holon identifier
+   * @param {string} lens - Lens to analyze
+   * @param {Object} timeRange - Time range {start, end}
+   * @returns {Promise<Object>} Trend analysis
+   */
+  async spatialTrends(holon, lens, timeRange = {}) {
+    if (!this.holosphere) {
+      throw new Error('HoloSphere instance required');
+    }
+
+    const data = await this.holosphere.getAll(holon, lens);
+
+    // Filter by time if timestamps available
+    let filteredData = data;
+    if (timeRange.start || timeRange.end) {
+      filteredData = data.filter(item => {
+        const ts = item.timestamp || item.created_at || item.date;
+        if (!ts) return true;
+
+        const itemTime = new Date(ts).getTime();
+        if (timeRange.start && itemTime < new Date(timeRange.start).getTime()) return false;
+        if (timeRange.end && itemTime > new Date(timeRange.end).getTime()) return false;
+        return true;
+      });
+    }
+
+    const systemPrompt = `You are a trend analyst. Analyze temporal patterns in geographic data.
+
+Region: ${holon}
+Lens: ${lens}
+Time range: ${JSON.stringify(timeRange)}
+Items analyzed: ${filteredData.length}
+
+Identify:
+1. Growth/decline trends
+2. Seasonal patterns
+3. Emerging themes
+4. Predictions
+
+Return JSON:
+{
+  "trends": [{"trend": "description", "direction": "up|down|stable", "strength": 0.0-1.0}],
+  "patterns": ["pattern1"],
+  "emerging_themes": ["theme1"],
+  "predictions": ["pred1"],
+  "summary": "overview"
+}`;
+
+    const trends = await this.llm.getJSON(
+      systemPrompt,
+      JSON.stringify(filteredData.slice(0, 50), null, 2),
+      { temperature: 0.3 }
+    );
+
+    return {
+      holon,
+      lens,
+      timeRange,
+      dataCount: filteredData.length,
+      trends,
+      timestamp: Date.now(),
+    };
+  }
+
+  /**
+   * Find hotspots in hierarchical data
+   * @param {string} holon - Starting holon
+   * @param {string} lens - Lens to analyze
+   * @param {string} metric - Metric to identify hotspots by
+   * @returns {Promise<Object>} Hotspot analysis
+   */
+  async findHotspots(holon, lens, metric = 'count') {
+    if (!this.holosphere) {
+      throw new Error('HoloSphere instance required');
+    }
+
+    // Get data from holon and children
+    const data = await this.holosphere.getAll(holon, lens);
+
+    const systemPrompt = `You are a spatial analyst. Identify hotspots (areas of high concentration/activity) in geographic data.
+
+Region: ${holon}
+Lens: ${lens}
+Metric: ${metric}
+Data points: ${data.length}
+
+Identify:
+1. High-activity areas
+2. Emerging areas
+3. Underserved areas
+4. Recommended focus areas
+
+Return JSON:
+{
+  "hotspots": [{"location": "desc", "intensity": 0.0-1.0, "type": "high_activity|emerging|underserved"}],
+  "concentration_patterns": ["pattern1"],
+  "recommendations": ["rec1"],
+  "summary": "overview"
+}`;
+
+    return this.llm.getJSON(
+      systemPrompt,
+      JSON.stringify(data.slice(0, 50), null, 2),
+      { temperature: 0.3 }
+    );
+  }
+
+  /**
+   * Generate spatial report for a region
+   * @param {string} holon - Holon identifier
+   * @param {Object} options - Report options
+   * @returns {Promise<string>} Formatted report
+   */
+  async generateReport(holon, options = {}) {
+    if (!this.holosphere) {
+      throw new Error('HoloSphere instance required');
+    }
+
+    const lenses = options.lenses || ['default'];
+    const allData = {};
+
+    for (const lens of lenses) {
+      allData[lens] = await this.holosphere.getAll(holon, lens);
+    }
+
+    const systemPrompt = `You are a regional analyst. Generate a comprehensive report for this geographic region.
+
+Region: ${holon}
+Data by lens: ${Object.entries(allData).map(([l, d]) => `${l}: ${d.length} items`).join(', ')}
+
+Generate a professional report including:
+1. Executive Summary
+2. Key Statistics
+3. Notable Patterns
+4. Opportunities
+5. Recommendations
+
+Format as markdown.`;
+
+    const dataSample = {};
+    for (const [lens, data] of Object.entries(allData)) {
+      dataSample[lens] = data.slice(0, 20);
+    }
+
+    return this.llm.sendMessage(
+      systemPrompt,
+      JSON.stringify(dataSample, null, 2),
+      { temperature: 0.5, maxTokens: 2000 }
+    );
+  }
+}
+
+export default SpatialAnalysis;

package/src/ai/tts.js

@@ -0,0 +1,194 @@
+/**
+ * Text-to-Speech - OpenAI TTS integration
+ */
+
+/**
+ * Available voices for TTS
+ */
+export const VOICES = {
+  ALLOY: 'alloy',
+  ECHO: 'echo',
+  FABLE: 'fable',
+  ONYX: 'onyx',
+  NOVA: 'nova',
+  SHIMMER: 'shimmer',
+};
+
+/**
+ * Available TTS models
+ */
+export const MODELS = {
+  TTS_1: 'tts-1',
+  TTS_1_HD: 'tts-1-hd',
+};
+
+/**
+ * TTS class for text-to-speech conversion
+ */
+export class TTS {
+  /**
+   * @param {Object} openai - OpenAI client instance
+   */
+  constructor(openai) {
+    this.openai = openai;
+    this.defaultVoice = VOICES.ECHO;
+    this.defaultModel = MODELS.TTS_1;
+  }
+
+  /**
+   * Convert text to speech
+   * @param {string} text - Text to convert
+   * @param {Object} options - TTS options
+   * @returns {Promise<Buffer>} Audio buffer (MP3)
+   */
+  async speak(text, options = {}) {
+    const {
+      voice = this.defaultVoice,
+      model = this.defaultModel,
+      speed = 1.0,
+      responseFormat = 'mp3',
+    } = options;
+
+    try {
+      const response = await this.openai.audio.speech.create({
+        model,
+        voice,
+        input: text,
+        speed,
+        response_format: responseFormat,
+      });
+
+      // Convert response to Buffer
+      const arrayBuffer = await response.arrayBuffer();
+      return Buffer.from(arrayBuffer);
+    } catch (error) {
+      throw new Error(`TTS failed: ${error.message}`);
+    }
+  }
+
+  /**
+   * Convert text to speech and return as base64
+   * @param {string} text - Text to convert
+   * @param {Object} options - TTS options
+   * @returns {Promise<string>} Base64 encoded audio
+   */
+  async speakBase64(text, options = {}) {
+    const buffer = await this.speak(text, options);
+    return buffer.toString('base64');
+  }
+
+  /**
+   * Convert text to speech and return as data URL
+   * @param {string} text - Text to convert
+   * @param {Object} options - TTS options
+   * @returns {Promise<string>} Data URL for audio
+   */
+  async speakDataUrl(text, options = {}) {
+    const format = options.responseFormat || 'mp3';
+    const mimeType = format === 'mp3' ? 'audio/mpeg' : `audio/${format}`;
+    const base64 = await this.speakBase64(text, options);
+    return `data:${mimeType};base64,${base64}`;
+  }
+
+  /**
+   * Set default voice
+   * @param {string} voice - Voice name
+   */
+  setDefaultVoice(voice) {
+    if (!Object.values(VOICES).includes(voice)) {
+      throw new Error(`Invalid voice: ${voice}. Use one of: ${Object.values(VOICES).join(', ')}`);
+    }
+    this.defaultVoice = voice;
+  }
+
+  /**
+   * Set default model
+   * @param {string} model - Model name
+   */
+  setDefaultModel(model) {
+    if (!Object.values(MODELS).includes(model)) {
+      throw new Error(`Invalid model: ${model}. Use one of: ${Object.values(MODELS).join(', ')}`);
+    }
+    this.defaultModel = model;
+  }
+
+  /**
+   * Get available voices
+   * @returns {string[]} Available voice names
+   */
+  static getVoices() {
+    return Object.values(VOICES);
+  }
+
+  /**
+   * Get available models
+   * @returns {string[]} Available model names
+   */
+  static getModels() {
+    return Object.values(MODELS);
+  }
+
+  /**
+   * Estimate audio duration (approximate)
+   * @param {string} text - Text to estimate
+   * @param {number} speed - Speech speed (default: 1.0)
+   * @returns {number} Estimated duration in seconds
+   */
+  static estimateDuration(text, speed = 1.0) {
+    // Average speaking rate: ~150 words per minute
+    const words = text.split(/\s+/).length;
+    const minutes = words / 150;
+    return (minutes * 60) / speed;
+  }
+
+  /**
+   * Split text into chunks for long content
+   * @param {string} text - Long text
+   * @param {number} maxChars - Max characters per chunk (default: 4000)
+   * @returns {string[]} Text chunks
+   */
+  static splitText(text, maxChars = 4000) {
+    if (text.length <= maxChars) {
+      return [text];
+    }
+
+    const chunks = [];
+    const sentences = text.split(/(?<=[.!?])\s+/);
+    let currentChunk = '';
+
+    for (const sentence of sentences) {
+      if ((currentChunk + sentence).length > maxChars) {
+        if (currentChunk) {
+          chunks.push(currentChunk.trim());
+        }
+        currentChunk = sentence;
+      } else {
+        currentChunk += (currentChunk ? ' ' : '') + sentence;
+      }
+    }
+
+    if (currentChunk) {
+      chunks.push(currentChunk.trim());
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Convert long text to speech (handles chunking)
+   * @param {string} text - Long text
+   * @param {Object} options - TTS options
+   * @returns {Promise<Buffer[]>} Array of audio buffers
+   */
+  async speakLong(text, options = {}) {
+    const chunks = TTS.splitText(text, options.maxChars || 4000);
+
+    const buffers = await Promise.all(
+      chunks.map(chunk => this.speak(chunk, options))
+    );
+
+    return buffers;
+  }
+}
+
+export default TTS;