@sixthsummit/ship-api 0.0.0-stage → 4.1.1

package/lib/libraries/weaviate/schemas/SchemaRegistry.ts

@@ -0,0 +1,284 @@
+import Logger from "@sixthsummit/ship-api/app/utils/Logger";
+import type { Plugin, SchemaConfig, SearchMethod } from "../types/index.js";
+
+export default class SchemaRegistry {
+  private static instance: SchemaRegistry;
+  private schemas: Map<string, SchemaConfig> = new Map();
+  private plugins: Map<string, Plugin> = new Map();
+  private searchMethods: Map<string, SearchMethod> = new Map();
+
+  constructor(schemas: SchemaConfig[] = []) {
+    this.registerDefaultSearchMethods();
+
+    // Register provided schemas
+    schemas.forEach((schema) => {
+      this.registerSchema(schema);
+    });
+  }
+
+  /**
+   * Initialize the static registry with schemas
+   */
+  static initialize(schemas: SchemaConfig[]): void {
+    SchemaRegistry.instance = new SchemaRegistry(schemas);
+    Logger.log(`Static SchemaRegistry initialized with ${schemas.length} schemas`);
+  }
+
+  /**
+   * Get the static registry instance
+   */
+  static getInstance(): SchemaRegistry {
+    if (!SchemaRegistry.instance) {
+      throw new Error("SchemaRegistry not initialized. Call SchemaRegistry.initialize() first.");
+    }
+    return SchemaRegistry.instance;
+  }
+
+  /**
+   * Check if the registry is initialized
+   */
+  static isInitialized(): boolean {
+    return SchemaRegistry.instance !== undefined;
+  }
+
+  /**
+   * Register a schema configuration
+   * Validates the schema before registration (fail-fast principle)
+   */
+  registerSchema(schema: SchemaConfig): void {
+    // Validate schema before registration
+    const validation = this.validateSchema(schema);
+    if (!validation.valid) {
+      throw new Error(
+        `Schema validation failed for '${schema.name}': ${validation.errors.join(", ")}`,
+      );
+    }
+
+    this.schemas.set(schema.name, schema);
+    Logger.log(`Schema registered: ${schema.name}`);
+  }
+
+  /**
+   * Register multiple schemas
+   */
+  registerSchemas(schemas: Record<string, SchemaConfig>): void {
+    Object.values(schemas).forEach((schema) => {
+      this.registerSchema(schema);
+    });
+  }
+
+  /**
+   * Get a schema by name
+   */
+  getSchema(name: string): SchemaConfig | undefined {
+    return this.schemas.get(name);
+  }
+
+  /**
+   * Get all registered schemas
+   */
+  getAllSchemas(): SchemaConfig[] {
+    return Array.from(this.schemas.values());
+  }
+
+  /**
+   * Check if a schema exists
+   */
+  hasSchema(name: string): boolean {
+    return this.schemas.has(name);
+  }
+
+  /**
+   * Get schema names
+   */
+  getSchemaNames(): string[] {
+    return Array.from(this.schemas.keys());
+  }
+
+  /**
+   * Register a plugin
+   */
+  registerPlugin(plugin: Plugin): void {
+    this.plugins.set(plugin.name, plugin);
+    Logger.log(`Plugin registered: ${plugin.name} v${plugin.version}`);
+  }
+
+  /**
+   * Get a plugin by name
+   */
+  getPlugin(name: string): Plugin | undefined {
+    return this.plugins.get(name);
+  }
+
+  /**
+   * Get plugins for a schema
+   */
+  getSchemaPlugins(schemaName: string): Plugin[] {
+    const schema = this.getSchema(schemaName);
+    if (!schema?.plugins) return [];
+
+    return schema.plugins
+      .map((pluginName) => this.getPlugin(pluginName))
+      .filter(Boolean) as Plugin[];
+  }
+
+  /**
+   * Register a search method
+   */
+  registerSearchMethod(method: SearchMethod): void {
+    this.searchMethods.set(method.name, method);
+    Logger.log(`Search method registered: ${method.name}`);
+  }
+
+  /**
+   * Get a search method by name
+   */
+  getSearchMethod(name: string): SearchMethod | undefined {
+    return this.searchMethods.get(name);
+  }
+
+  /**
+   * Get all search methods
+   */
+  getAllSearchMethods(): SearchMethod[] {
+    return Array.from(this.searchMethods.values());
+  }
+
+  /**
+   * Get searchable fields for a schema
+   */
+  getSearchableFields(schemaName: string): string[] {
+    const schema = this.getSchema(schemaName);
+    return schema?.searchableFields || [];
+  }
+
+  /**
+   * Get filterable fields for a schema
+   */
+  getFilterableFields(schemaName: string): string[] {
+    const schema = this.getSchema(schemaName);
+    return schema?.filterableFields || [];
+  }
+
+  /**
+   * Get sortable fields for a schema
+   */
+  getSortableFields(schemaName: string): string[] {
+    const schema = this.getSchema(schemaName);
+    return schema?.sortableFields || [];
+  }
+
+  /**
+   * Validate a schema configuration
+   */
+  validateSchema(schema: SchemaConfig): { valid: boolean; errors: string[] } {
+    const errors: string[] = [];
+
+    if (!schema.name) {
+      errors.push("Schema name is required");
+    }
+
+    if (!schema.properties || schema.properties.length === 0) {
+      errors.push("Schema must have at least one property");
+    }
+
+    if (schema.vectorizer && schema.vectorizer !== "none" && !schema.vectorizer.name) {
+      errors.push("Vectorizer name is required if vectorizer is specified");
+    }
+
+    // Validate properties
+    schema.properties?.forEach((prop, index) => {
+      if (!prop.name) {
+        errors.push(`Property ${index} must have a name`);
+      }
+      if (!prop.dataType || prop.dataType.length === 0) {
+        errors.push(`Property ${prop.name} must have at least one data type`);
+      }
+    });
+
+    return {
+      valid: errors.length === 0,
+      errors,
+    };
+  }
+
+  /**
+   * Get schema statistics
+   */
+  getStats(): {
+    totalSchemas: number;
+    totalPlugins: number;
+    totalSearchMethods: number;
+    schemasByCategory: Record<string, number>;
+  } {
+    const schemas = this.getAllSchemas();
+    const schemasByCategory = schemas.reduce(
+      (acc, schema) => {
+        const category = schema.metadata?.category || "unknown";
+        acc[category] = (acc[category] || 0) + 1;
+        return acc;
+      },
+      {} as Record<string, number>,
+    );
+
+    return {
+      totalSchemas: this.schemas.size,
+      totalPlugins: this.plugins.size,
+      totalSearchMethods: this.searchMethods.size,
+      schemasByCategory,
+    };
+  }
+
+  /**
+   * Register default search methods
+   */
+  private registerDefaultSearchMethods(): void {
+    // Vector search method
+    this.registerSearchMethod({
+      name: "vector",
+      description: "Vector-based semantic search",
+      supportedOptions: [
+        "nearText",
+        "nearVector",
+        "nearObject",
+        "where",
+        "limit",
+        "offset",
+        "sort",
+      ],
+      execute: async () => {
+        throw new Error("Vector search method must be implemented by the service");
+      },
+    });
+
+    // BM25 search method
+    this.registerSearchMethod({
+      name: "bm25",
+      description: "BM25 keyword-based search",
+      supportedOptions: ["bm25", "where", "limit", "offset", "sort"],
+      execute: async () => {
+        throw new Error("BM25 search method must be implemented by the service");
+      },
+    });
+
+    // Hybrid search method
+    this.registerSearchMethod({
+      name: "hybrid",
+      description: "Hybrid search combining vector and BM25",
+      supportedOptions: ["hybrid", "where", "limit", "offset", "sort"],
+      execute: async () => {
+        throw new Error("Hybrid search method must be implemented by the service");
+      },
+    });
+  }
+
+  /**
+   * Clear all registrations
+   */
+  clear(): void {
+    this.schemas.clear();
+    this.plugins.clear();
+    this.searchMethods.clear();
+    Logger.log("Schema registry cleared");
+  }
+}

package/lib/libraries/weaviate/schemas/index.ts

@@ -0,0 +1,2 @@
+// Export SchemaRegistry
+export { default as SchemaRegistry } from "./SchemaRegistry.js";

package/lib/libraries/weaviate/types/BM25SearchOptions.ts

@@ -0,0 +1,9 @@
+import type { SearchOptions } from "./SearchOptions.js";
+
+export interface BM25SearchOptions extends SearchOptions {
+  bm25?: {
+    query: string;
+    properties?: string[];
+    boost?: number;
+  };
+}

package/lib/libraries/weaviate/types/ExtendedWeaviateClient.ts

@@ -0,0 +1,7 @@
+import type { WeaviateClient } from "weaviate-client";
+
+/**
+ * Extended WeaviateClient type
+ * Note: v3 client has proper typing, so this just re-exports WeaviateClient for compatibility
+ */
+export type ExtendedWeaviateClient = WeaviateClient;

package/lib/libraries/weaviate/types/HybridSearchOptions.ts

@@ -0,0 +1,11 @@
+import type { SearchOptions } from "./SearchOptions.js";
+
+export interface HybridSearchOptions extends SearchOptions {
+  hybrid?: {
+    query: string;
+    alpha?: number; // 0 = pure BM25, 1 = pure vector
+    properties?: string[];
+    vector?: number[];
+    fusionType?: "RelativeScore" | "Ranked"; // How to combine BM25 + vector scores (Weaviate values)
+  };
+}

package/lib/libraries/weaviate/types/MultiTenancy.ts

@@ -0,0 +1,81 @@
+/**
+ * Multi-tenancy configuration for Weaviate collections
+ */
+export interface MultiTenancyConfig {
+  enabled: boolean;
+  autoTenantCreation?: boolean;
+  autoTenantActivation?: boolean;
+}
+
+/**
+ * Weaviate v3 tenant activity status values
+ */
+export type TenantActivityStatus =
+  | "HOT"
+  | "COLD"
+  | "FROZEN"
+  | "INACTIVE"
+  | "OFFLOADED"
+  | "OFFLOADING"
+  | "ONLOADING";
+
+/**
+ * Tenant information
+ */
+export interface Tenant {
+  name: string;
+  activityStatus: TenantActivityStatus;
+  createdAt?: string;
+  updatedAt?: string;
+}
+
+/**
+ * Tenant creation options
+ */
+export interface CreateTenantOptions {
+  name: string;
+  activityStatus?: "HOT" | "COLD" | "FROZEN";
+}
+
+/**
+ * Tenant update options
+ */
+export interface UpdateTenantOptions {
+  activityStatus?: "HOT" | "COLD" | "FROZEN";
+}
+
+/**
+ * Tenant context for operations
+ */
+export interface TenantContext {
+  tenant: string;
+  activityStatus?: "HOT" | "COLD" | "FROZEN";
+}
+
+/**
+ * Multi-tenant search options
+ */
+export interface MultiTenantSearchOptions {
+  tenant?: string;
+  tenants?: string[];
+  includeInactive?: boolean;
+}
+
+/**
+ * Tenant statistics
+ */
+export interface TenantStats {
+  totalTenants: number;
+  activeTenants: number;
+  inactiveTenants: number;
+  coldTenants: number;
+}
+
+/**
+ * Tenant management result
+ */
+export interface TenantManagementResult {
+  success: boolean;
+  tenant?: Tenant;
+  error?: string;
+}

package/lib/libraries/weaviate/types/Plugin.ts

@@ -0,0 +1,17 @@
+import type { SearchResult } from "./SearchResult.js";
+
+export interface Plugin {
+  name: string;
+  version: string;
+  description: string;
+  hooks: {
+    beforeSearch?: (className: string, options: any) => any;
+    afterSearch?: (className: string, result: SearchResult) => SearchResult;
+    beforeCreate?: (className: string, data: any) => any;
+    afterCreate?: (className: string, result: any) => any;
+    beforeUpdate?: (className: string, id: string, data: any) => any;
+    afterUpdate?: (className: string, result: any) => any;
+    beforeDelete?: (className: string, id: string) => any;
+    afterDelete?: (className: string, result: any) => any;
+  };
+}

package/lib/libraries/weaviate/types/SchemaConfig.ts

@@ -0,0 +1,25 @@
+import type { MultiTenancyConfig } from "./MultiTenancy.js";
+import type { SearchableField } from "./SearchableField.js";
+import type { VectorIndexConfig } from "./VectorIndexConfig.js";
+import type { Vectorizer } from "./VectorizerConfig.js";
+
+export interface SchemaConfig {
+  name: string;
+  description?: string;
+  vectorizer?: Vectorizer;
+  vectorIndexConfig?: VectorIndexConfig;
+  properties: SearchableField[];
+  searchableFields?: string[]; // Fields to include in search by default
+  filterableFields?: string[]; // Fields that can be used in where clauses
+  sortableFields?: string[]; // Fields that can be used for sorting
+  relationships?: {
+    [fieldName: string]: {
+      targetClass: string;
+      type: "one-to-one" | "one-to-many" | "many-to-one" | "many-to-many";
+      required?: boolean;
+    };
+  };
+  plugins?: string[]; // Custom plugin names to apply
+  metadata?: Record<string, any>; // Additional metadata
+  multiTenancy?: MultiTenancyConfig; // Multi-tenancy configuration
+}

package/lib/libraries/weaviate/types/SearchMethod.ts

@@ -0,0 +1,8 @@
+import type { SearchResult } from "./SearchResult.js";
+
+export interface SearchMethod {
+  name: string;
+  description: string;
+  supportedOptions: string[];
+  execute: (className: string, options: any) => Promise<SearchResult>;
+}

package/lib/libraries/weaviate/types/SearchOptions.ts

@@ -0,0 +1,16 @@
+import type { TenantContext } from "./MultiTenancy.js";
+
+export interface SearchOptions {
+  query?: string;
+  limit?: number;
+  offset?: number;
+  where?: Record<string, any>;
+  sort?: Array<{
+    field: string;
+    order: "asc" | "desc";
+  }>;
+  fields?: string[]; // Specific fields to return
+  includeVector?: boolean;
+  includeMetadata?: boolean;
+  tenant?: TenantContext; // Tenant context for multi-tenant operations
+}

package/lib/libraries/weaviate/types/SearchRequest.ts

@@ -0,0 +1,19 @@
+import type { BM25SearchOptions, HybridSearchOptions, VectorSearchOptions } from "./index.js";
+
+/**
+ * Discriminated union for search requests
+ * This provides better type safety than method overloads
+ */
+export type SearchRequest =
+  | {
+      method: "vector";
+      options: VectorSearchOptions;
+    }
+  | {
+      method: "bm25";
+      options: BM25SearchOptions;
+    }
+  | {
+      method: "hybrid";
+      options: HybridSearchOptions;
+    };

package/lib/libraries/weaviate/types/SearchResult.ts

@@ -0,0 +1,16 @@
+export interface SearchResult<T = any> {
+  objects: Array<{
+    id: string;
+    properties: T;
+    vector?: number[];
+    certainty?: number;
+    distance?: number;
+    score?: number;
+    metadata?: {
+      explainScore?: string;
+      [key: string]: any;
+    };
+  }>;
+  total?: number;
+  took?: number;
+}

package/lib/libraries/weaviate/types/SearchableField.ts

@@ -0,0 +1,87 @@
+/**
+ * Configuration interface for Weaviate searchable fields
+ *
+ * @example
+ * ```typescript
+ * // High-priority search field
+ * {
+ *   name: "fullName",
+ *   dataType: ["string"],
+ *   description: "Full name of the person",
+ *   indexInverted: true,
+ *   indexSearchable: true,
+ *   weight: 2.0,
+ *   boost: 1.5
+ * }
+ *
+ * // Filter-only field
+ * {
+ *   name: "city",
+ *   dataType: ["string"],
+ *   description: "City of residence",
+ *   indexInverted: true,
+ *   indexFilterable: true,
+ *   weight: 0.5
+ * }
+ * ```
+ */
+export interface SearchableField {
+  /** The name of the field in the Weaviate schema */
+  name: string;
+
+  /** Array of data types that this field can contain (e.g., ['string', 'text']) */
+  dataType: string[];
+
+  /** Optional human-readable description of what this field represents */
+  description?: string;
+
+  /**
+   * Whether to create an inverted index for this field (enables full-text search)
+   * @default true
+   */
+  indexInverted?: boolean;
+
+  /**
+   * Whether this field can be used in filter operations (WHERE clauses)
+   * @default true
+   */
+  indexFilterable?: boolean;
+
+  /**
+   * Whether this field is included in search operations and can be searched against
+   * @default true
+   */
+  indexSearchable?: boolean;
+
+  /**
+   * Weight for BM25 scoring algorithm - higher values make this field more important in text search relevance
+   * @default 1.0
+   * @example
+   * - 2.0: High importance (names, titles)
+   * - 1.5: Medium-high importance (normalized names)
+   * - 1.0: Normal importance (descriptions, bios)
+   * - 0.5: Low importance (metadata, locations)
+   * - 0.3: Very low importance (codes, IDs)
+   */
+  weight?: number;
+
+  /**
+   * Boost factor for vector search - multiplies the similarity score for this field in vector-based searches
+   * @default 1.0
+   * @example
+   * - 1.5: High vector search boost (primary search fields)
+   * - 1.2: Medium vector search boost (secondary fields)
+   * - 1.0: No boost (default)
+   */
+  boost?: number;
+
+  /**
+   * Tokenization method for text fields in Weaviate
+   * @example
+   * - "field": Tokenize the entire field as a single token
+   * - "word": Tokenize by words (default)
+   * - "lowercase": Convert to lowercase before tokenization
+   * - "whitespace": Split on whitespace
+   */
+  tokenization?: "field" | "word" | "lowercase" | "whitespace";
+}

package/lib/libraries/weaviate/types/VectorIndexConfig.ts

@@ -0,0 +1,19 @@
+export interface VectorIndexConfig {
+  distance?: "cosine" | "dot" | "l2-squared" | "hamming" | "manhattan";
+  ef?: number;
+  efConstruction?: number;
+  maxConnections?: number;
+  efSearch?: number;
+  skip?: boolean;
+  cleanupIntervalSeconds?: number;
+  pq?: {
+    enabled?: boolean;
+    segments?: number;
+    centroids?: number;
+    trainingLimit?: number;
+    encoder?: {
+      type?: string;
+      distribution?: string;
+    };
+  };
+}

package/lib/libraries/weaviate/types/VectorSearchOptions.ts

@@ -0,0 +1,18 @@
+import type { SearchOptions } from "./SearchOptions.js";
+
+export interface VectorSearchOptions extends SearchOptions {
+  nearText?: {
+    concepts: string[];
+    certainty?: number;
+    moveAwayFrom?: { concepts: string[]; force: number };
+    moveTo?: { concepts: string[]; force: number };
+  };
+  nearVector?: {
+    vector: number[];
+    certainty?: number;
+  };
+  nearObject?: {
+    id: string;
+    certainty?: number;
+  };
+}

package/lib/libraries/weaviate/types/VectorizerConfig.ts

@@ -0,0 +1,8 @@
+export interface VectorizerConfig {
+  name: string;
+  model?: string;
+  dimensions?: number;
+  parameters?: Record<string, any>;
+}
+
+export type Vectorizer = VectorizerConfig | "none";

package/lib/libraries/weaviate/types/index.ts

@@ -0,0 +1,29 @@
+// Schema-related types
+
+export type { BM25SearchOptions } from "./BM25SearchOptions.js";
+export type { ExtendedWeaviateClient } from "./ExtendedWeaviateClient.js";
+export type { HybridSearchOptions } from "./HybridSearchOptions.js";
+// Multi-tenancy types
+export type {
+  CreateTenantOptions,
+  MultiTenancyConfig,
+  MultiTenantSearchOptions,
+  Tenant,
+  TenantActivityStatus,
+  TenantContext,
+  TenantManagementResult,
+  TenantStats,
+  UpdateTenantOptions,
+} from "./MultiTenancy.js";
+// Plugin types
+export type { Plugin } from "./Plugin.js";
+export type { SchemaConfig } from "./SchemaConfig.js";
+export type { SearchableField } from "./SearchableField.js";
+export type { SearchMethod } from "./SearchMethod.js";
+// Search-related types
+export type { SearchOptions } from "./SearchOptions.js";
+export type { SearchRequest } from "./SearchRequest.js";
+export type { SearchResult } from "./SearchResult.js";
+export type { VectorIndexConfig } from "./VectorIndexConfig.js";
+export type { Vectorizer, VectorizerConfig } from "./VectorizerConfig.js";
+export type { VectorSearchOptions } from "./VectorSearchOptions.js";