@bett3r-dev/pv3 1.0.0

package/src/ports/infrastructure/__tests__/fixture/structureState.ts

@@ -0,0 +1,254 @@
+export const structureState = {
+  state: {
+    brokers: {
+      '10.0.0.13': {
+        subscriptions: {
+          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
+            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-0': {
+              host: '10.0.0.13',
+              index: 0,
+            },
+          },
+          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
+            'ArithmeticOperationsAggregate-TotalByColorReadModel-0': {
+              host: '10.0.0.13',
+              index: 0,
+            },
+          },
+        },
+      },
+      '10.0.0.14': {
+        subscriptions: {
+          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
+            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-1': {
+              host: '10.0.0.14',
+              index: 1,
+            },
+          },
+          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
+            'ArithmeticOperationsAggregate-TotalByColorReadModel-1': {
+              host: '10.0.0.14',
+              index: 1,
+            },
+          },
+        },
+      },
+      '10.0.0.15': {
+        subscriptions: {
+          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
+            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-2': {
+              host: '10.0.0.15',
+              index: 2,
+            },
+          },
+          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
+            'ArithmeticOperationsAggregate-TotalByColorReadModel-2': {
+              host: '10.0.0.15',
+              index: 2,
+            },
+          },
+        },
+      },
+      '10.0.0.16': {
+        subscriptions: {
+          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
+            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-3': {
+              host: '10.0.0.16',
+              index: 3,
+            },
+          },
+          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
+            'ArithmeticOperationsAggregate-TotalByColorReadModel-3': {
+              host: '10.0.0.16',
+              index: 3,
+            },
+          },
+        },
+      },
+      '10.0.0.18': {
+        subscriptions: {
+          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
+            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-4': {
+              host: '10.0.0.18',
+              index: 4,
+            },
+          },
+          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
+            'ArithmeticOperationsAggregate-TotalByColorReadModel-4': {
+              host: '10.0.0.18',
+              index: 4,
+            },
+          },
+        },
+      },
+      '10.0.0.19': {
+        subscriptions: {
+          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
+            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-5': {
+              host: '10.0.0.19',
+              index: 5,
+            },
+          },
+          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
+            'ArithmeticOperationsAggregate-TotalByColorReadModel-5': {
+              host: '10.0.0.19',
+              index: 5,
+            },
+          },
+        },
+      },
+      '10.0.0.20': {
+        subscriptions: {
+          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
+            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-6': {
+              host: '10.0.0.20',
+              index: 6,
+            },
+          },
+          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
+            'ArithmeticOperationsAggregate-TotalByColorReadModel-6': {
+              host: '10.0.0.20',
+              index: 6,
+            },
+          },
+        },
+      },
+      '10.0.0.21': {
+        subscriptions: {
+          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
+            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-7': {
+              host: '10.0.0.21',
+              index: 7,
+            },
+          },
+          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
+            'ArithmeticOperationsAggregate-TotalByColorReadModel-7': {
+              host: '10.0.0.21',
+              index: 7,
+            },
+          },
+        },
+      },
+    },
+    services: {
+      service0: {
+        port: 2013,
+        consumes: {
+          OperationCountByColorReadModel: [
+            'DividedBy_17',
+            'MultipliedBy_18',
+            'MultipliedBy_2',
+            'MultipliedBy_5',
+            'Subtracted_50',
+            'Added_20',
+            'Added_7',
+            'Added_3',
+            'DividedBy_3',
+          ],
+          TotalByColorReadModel: [
+            'MultipliedBy_18',
+            'DividedBy_17',
+            'MultipliedBy_5',
+            'MultipliedBy_2',
+            'Subtracted_50',
+            'Added_20',
+            'Added_7',
+            'Added_3',
+            'DividedBy_3',
+          ],
+        },
+        produces: {
+          ArithmeticOperationsAggregate: [
+            'DividedBy_17',
+            'Added_20',
+            'MultipliedBy_5',
+            'Added_3',
+            'Added_7',
+            'MultipliedBy_2',
+            'Subtracted_50',
+            'DividedBy_3',
+            'MultipliedBy_18',
+          ],
+        },
+        isBroker: true,
+        isService: true,
+        endpoints: [
+          'GET:/arithmetic-operations-aggregate/{id}',
+          'GET:/arithmetic-operations-aggregate/{id}/stream',
+          'POST:/arithmetic-operations-aggregate/{id}/divide-by-17',
+          'POST:/arithmetic-operations-aggregate/{id}/add-20',
+          'POST:/arithmetic-operations-aggregate/{id}/multiply-by-5',
+          'POST:/arithmetic-operations-aggregate/{id}/add-3',
+          'POST:/arithmetic-operations-aggregate/{id}/add-7',
+          'POST:/arithmetic-operations-aggregate/{id}/multiply-by-2',
+          'POST:/arithmetic-operations-aggregate/{id}/subtract-50',
+          'POST:/arithmetic-operations-aggregate/{id}/divide-by-3',
+          'POST:/arithmetic-operations-aggregate/{id}/multiply-by-18',
+          'POST:/operation-count-by-color-read-model',
+          'GET:/operation-count-by-color-read-model/{id}?',
+          'POST:/total-by-color-read-model',
+          'GET:/total-by-color-read-model/{id}?',
+        ],
+        possibleSubscriptions: [
+          'ArithmeticOperationsAggregate-OperationCountByColorReadModel',
+          'ArithmeticOperationsAggregate-TotalByColorReadModel',
+        ],
+        hosts: [
+          '10.0.0.13',
+          '10.0.0.14',
+          '10.0.0.15',
+          '10.0.0.16',
+          '10.0.0.18',
+          '10.0.0.19',
+          '10.0.0.20',
+          '10.0.0.21',
+        ],
+      },
+    },
+    subscriptions: {
+      'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
+        consumer: 'OperationCountByColorReadModel',
+        endpoint: '/operation-count-by-color-read-model',
+        events: [
+          'DividedBy_17',
+          'MultipliedBy_18',
+          'MultipliedBy_2',
+          'MultipliedBy_5',
+          'Subtracted_50',
+          'Added_20',
+          'Added_7',
+          'Added_3',
+          'DividedBy_3',
+        ],
+        producer: 'ArithmeticOperationsAggregate',
+        dataSplits: {
+          manual: [],
+          custom: [],
+          automatic: [],
+        },
+      },
+      'ArithmeticOperationsAggregate-TotalByColorReadModel': {
+        consumer: 'OperationCountByColorReadModel',
+        endpoint: '/total-by-color-read-model',
+        events: [
+          'MultipliedBy_18',
+          'DividedBy_17',
+          'MultipliedBy_5',
+          'MultipliedBy_2',
+          'Subtracted_50',
+          'Added_20',
+          'Added_7',
+          'Added_3',
+          'DividedBy_3',
+        ],
+        producer: 'ArithmeticOperationsAggregate',
+        dataSplits: {
+          manual: [],
+          custom: [],
+          automatic: [],
+        },
+      },
+    },
+  },
+  expectedVersion: 8,
+};

package/src/ports/infrastructure/__tests__/infrastructure.test.ts

@@ -0,0 +1,11 @@
+import { getStructureDifferences } from '..';
+import { infrastructureServices, structureState } from './fixture';
+
+describe( 'Infrastructure Port', () => {
+  describe( 'getStructureDifferences', () => {
+    it( 'works', async () => {
+      // const result = getStructureDifferences( structureState, infrastructureServices );
+      // result;//?
+    });
+  });
+});

package/src/ports/infrastructure/index.ts

@@ -0,0 +1,162 @@
+import { reduce } from '@bett3r-dev/bett3r-utils';
+import { EnvironmentServiceType, InfrastructureAdapterType, InfrastructureConfigType, InfrastructurePortType, InfrastructureServiceType, InfrastructureThisServiceType, Ports, StructureServiceType } from '@bett3r-dev/pv3-types';
+import { OpenAPIV3_1 } from 'openapi-types';
+import { append, assoc, compose, converge, identity, omit } from 'ramda';
+import { LocalInfrastructure } from '../../adapters/infrastructure';
+import { ClusterStructureType, StructureState, UpsertHostCommandType } from '@bett3r-dev/pv3-types';
+import { createPort } from '../base';
+import { discoverSubscriptions, getServiceDefinition, getThisServiceIps } from './utils';
+export { getThisServiceIps } from './utils';
+
+export const getStructureDifferences = ( structureState: StructureState, infra: InfrastructureServiceType[]) => {
+  const serviceHostsSet = new Set( Object.values( structureState.state.services ).reduce(( hosts, service ) => hosts.concat( service.hosts ), [] as string[]));
+  const upsertHosts: UpsertHostCommandType[] = [];
+  const infraServices = infra.reduce(( services, service ) => {
+    if ( service.isService )
+      services[service.service] = service as any;
+    return services;
+  }, {} as Record<string, Omit<StructureServiceType, 'hosts'>> );
+  const { subscriptions, servicesSubscriptions } = discoverSubscriptions( infraServices );
+  for ( const service of infra ) {
+    for ( const { host } of service.instances ) {
+      serviceHostsSet.delete( host );
+      upsertHosts.push({
+        ...omit(['instances'], service ),
+        host,
+        possibleSubscriptions: servicesSubscriptions[service.service]
+      });
+    }
+  }
+  const removeHosts = Array.from( serviceHostsSet );
+  return {
+    upsertHosts,
+    removeHosts,
+    subscriptions
+  };
+};
+
+const checkInfrastructureIntegrity = ( structure: ClusterStructureType ) => async ( infra: InfrastructureServiceType[], retryBackoff = 100 ) => {
+  const structureState = await structure.getCurrentState();
+  const differences = getStructureDifferences( structureState, infra );
+  console.log( '$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ STRUCTURE DIFFERENCE Version', structureState.version, differences.upsertHosts.map( x => x.host ));
+  await structure.SetStructure( differences, structureState.version )
+    .catch(( e ) => {
+      if ( e.type !== 'ConcurrencyError' )
+        throw e;
+      else {
+        console.log( '$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ CONCURRENCY ERROR', e );
+        return new Promise(( resolve ) => {
+          setTimeout(() => {
+            checkInfrastructureIntegrity( structure )( infra, retryBackoff * 2 )
+              .then( resolve );
+          }, retryBackoff );
+        });
+      }
+    });
+  return infra;
+};
+
+const getCurrentServiceInfo = ( services: InfrastructureServiceType[]): InfrastructureThisServiceType => {
+  const ips = getThisServiceIps();
+
+  const brokerHosts = services
+    .reduce(( brokers, service, index ) =>{
+      if ( !service.isBroker ) return brokers;
+      brokers.push( ...service.instances.map( assoc( 'serviceIndex', index )));
+      return brokers;
+    }, [] as {host: string, timeRunning: number, serviceIndex:number}[])
+    .sort(( a, b ) => a.timeRunning - b.timeRunning );
+
+  const hostIndex = brokerHosts
+    .findIndex( host => ips.includes( host.host ));
+  const brokerHost = brokerHosts[hostIndex];
+
+  if ( hostIndex === -1 ) throw new Error( 'Service not found in infrastructure' );
+  return {
+    ...services[brokerHost.serviceIndex],
+    host: brokerHost.host,
+    leadershipRank: hostIndex,
+  };
+};
+
+export const getInfrastructureServicesDefinition = async ( infraServices: EnvironmentServiceType[]): Promise<InfrastructureServiceType[]> => {
+  return infraServices.map(( service ) => {
+    return {
+      service: service.service,
+      port: service.port,
+      instances: service.instances,
+      openApiEndpoint: service.openApiEndpoint,
+      ...( service.openApi ? getServiceDefinition( service.openApi ) : {})
+    } as InfrastructureServiceType;
+  });
+};
+
+
+export const InfrastructurePort = ( ports: Pick<Ports, 'logger'|'endpoints'|'configuration'>, config: InfrastructureConfigType, structure: ClusterStructureType, adapter?: InfrastructureAdapterType ): InfrastructurePortType => {
+  let servicesOpenApiDocumentation: Record<string, OpenAPIV3_1.Document> = {};
+  let _adapter: InfrastructureAdapterType;
+  let thisService: InfrastructureThisServiceType;
+  let adapters: InfrastructureAdapterType[] = append( LocalInfrastructure( ports ), adapter ? [adapter] : []);
+
+  const assignOpenApiDocumentation = compose(
+    ( services: any ) => servicesOpenApiDocumentation = services,
+    reduce(( docs, serviceDefinition ) => assoc( serviceDefinition.service, serviceDefinition.openApi, docs ), {})
+  );
+
+  const setOpenApiDocs = converge(( x, _ ) => x, [ identity, assignOpenApiDocumentation ]);
+
+  const start = async () => {
+    const _adapters = await Promise.allSettled( adapters!.map( adapter => adapter.start()));
+    const index = _adapters.findIndex( adapter => adapter.status === 'fulfilled' );
+    _adapter = adapters![index];
+    const checkInfrastructure = checkInfrastructureIntegrity( structure );
+    const infraServices = await _adapter.getServices().then( setOpenApiDocs ) as EnvironmentServiceType[];
+    const infraServicesDefinition = await getInfrastructureServicesDefinition( infraServices );
+    thisService = getCurrentServiceInfo( infraServicesDefinition );
+
+    _adapter.listenForInfrastructureChanges(( action, info ) => {
+      console.log( 'LOS SERVICES QUE RECIBI --------- ', info.instances.map( i=>i.host ));
+      if ( action === 'create' && info.openApi.tags?.find( tag => tag.name === 'pv3.outbox' )){
+        return;
+      }
+      //TODO: Crear una mejor manera de manejar el exponential backoff. y operaciones idempotentes
+      setTimeout(() => {
+        _adapter.getServices()
+          // .then(( s ) => {
+          //   const service = s.find( s=> s.service === info.service );
+          //   if ( service?.openApi.tags?.find( tag => tag.name === 'pv3.outbox' )){
+          //     return Promise.reject( 'IgnoringOutboxService' );
+          //   }
+          //   // return s.filter(( s ) => {
+          //   //   info.service === s.service
+          //   //   return s.openApi.tags?.find(( tag ) => tag.name !== 'pv3.outbox' );
+          //   // });
+          // })
+          .then( x=> { console.log( 'LOS SERVICES QUE VOY A AGREGAR --------- ', x.map( s=>s.instances.map( i=>i.host ))); return x; })
+          .then<EnvironmentServiceType[]>( setOpenApiDocs )
+          .then( getInfrastructureServicesDefinition )
+          .then( checkInfrastructure );
+      }, ( config.infrastructureLeaderRandomBackoff || 1000 ) * ( thisService.leadershipRank || 0 ));
+    });
+
+    await checkInfrastructure( infraServicesDefinition );
+    environments.isStarted = true;
+    environments.emitter.emit( 'started' );
+  };
+
+  const environments = createPort({
+    _adapter: () => _adapter,
+    name: 'Infrastructure',
+    getServicesOpenApiDocumentation: () => servicesOpenApiDocumentation,
+    getThisServiceInfo: () => thisService,
+    getServices: () => _adapter!.getServices(),
+    scaleService: ( service: string, replicasCount: number ) => _adapter!.scaleService( service, replicasCount ),
+    listenForInfrastructureChanges: ( callback: ( event: 'destroy' | 'create', service: EnvironmentServiceType ) => void ) => _adapter!.listenForInfrastructureChanges( callback ),
+    reloadLoadBalancer: () => _adapter!.reloadLoadBalancer(),
+    start,
+    stop: async () => _adapter.stop()
+  });
+
+  return environments;
+};
+

package/src/ports/infrastructure/utils.ts

@@ -0,0 +1,130 @@
+import { reduce } from '@bett3r-dev/bett3r-utils';
+import { StructureServiceType, StructureSubscriptionType } from '@bett3r-dev/pv3-types';
+import { paramCase } from 'change-case';
+import { OpenAPIV3_1 } from 'openapi-types';
+import os from 'os';
+import { omit } from 'ramda';
+
+export type DiscoveredSubscriptions = {
+  subscriptions: {
+    [id: string]: StructureSubscriptionType;
+  };
+  servicesSubscriptions: {
+    [host: string]: string[]
+  }
+}
+
+const extractPathEndpoints = ( path: OpenAPIV3_1.PathItemObject, url: string ) => {
+  return [ 'post', 'get', 'put', 'delete', 'patch', 'head', 'options' ]
+    .reduce(( endpoints, method ) => {
+      if ( path[method]) endpoints.push( `${method.toUpperCase()}:${url}` );
+      return endpoints;
+    }, [] as string[]);
+};
+
+
+export const getServiceDefinition = ( swagger: OpenAPIV3_1.Document ): StructureServiceType => {
+  const producerTags = [ 'aggregate', 'system' ];
+  const postSchemaProperty = 'responses';
+  const definition = Object.keys( swagger?.paths as OpenAPIV3_1.PathsObject || {}).reduce(( definition, url )=>{
+    const path = swagger.paths![url];
+    if ( path?.post?.tags?.some( t=> producerTags.includes( t ))){
+      const postSchemaBase = ( path.post as any )[postSchemaProperty];
+      const postSchema = postSchemaProperty === 'responses' ? postSchemaBase[200] : postSchemaBase;
+      const schema = postSchema.content['application/json'].schema;
+      let events;
+      if ( schema.anyOf ){
+        events = schema.anyOf
+          .map( s=>{
+            const ref = s.properties.data;
+            return ref.$ref.slice( ref.$ref.lastIndexOf( '/' ) + 1 );
+          });
+      } else if ( schema.properties?.events?.properties?.data?.anyOf ) {
+        events = schema.properties?.events?.properties?.data?.anyOf
+          .map( ref=>{
+            return ref.$ref.slice( ref.$ref.lastIndexOf( '/' ) + 1 );
+          });
+      } else {
+        events = [];
+      }
+      const handler = path.post?.tags?.filter( t=>!producerTags.includes( t ))[0];
+      definition.produces![handler] = [ ...( definition.produces![handler] || []), ...events ];
+    } else if ( path?.post?.tags?.some( t=> [ 'policy', 'readmodel' ].includes( t ))){
+      const events = ( path.post.requestBody as any ).content['application/json'].schema.anyOf.map( s=>{
+        const ref = s.properties.data;
+        return ref.$ref.slice( ref.$ref.lastIndexOf( '/' ) + 1 );
+      });
+      const handler = path.post?.tags?.filter( t=>![ 'policy', 'readmodel' ].includes( t ))[0];
+      definition.consumes![handler] = events;
+    }
+    return {
+      ...definition,
+      endpoints: [ ...definition.endpoints, ...extractPathEndpoints( path!, url ) ]
+    };
+  }, { consumes: {}, produces: {}, endpoints: []});
+  return {
+    ...omit(['endpoints'], definition ),
+    isBroker: swagger.tags?.some( t=>t.name === 'pv3.outbox' ),
+    isService: swagger.tags?.some( t=>t.name === 'pv3.service' ),
+    possibleModules: JSON.parse(( swagger.tags?.find( t=>t.name.startsWith( 'modules' )) || { name:'modules={}' }).name.split( 'modules=' )[1]),
+    endpoints: definition.endpoints
+  } as StructureServiceType;
+};
+
+export const discoverSubscriptions = ( services: Record<string, Omit<StructureServiceType, 'hosts'>> ): DiscoveredSubscriptions => {
+  const eventProducerMap = reduce(( eventMap, { produces }, service ) => {
+    const serviceMap = reduce(( eventMap, events, system: string ) => {
+      events.forEach( e=>{
+        eventMap[e] = { system, service: service };
+      });
+      return eventMap;
+    }, {} as Record<string, {system: string, service: string}>, produces );
+    return { ...eventMap, ...serviceMap };
+  } ,{} as Record<string, {system: string, service: string}>,services );
+
+  return reduce(( maps, { consumes, possibleModules }, service ) => {
+    const artifactModuleMap = Object.entries( possibleModules || {}).reduce(( acc, [ module, artifacts ]) => {
+      artifacts.forEach(( artifact ) => {
+        acc[artifact.split( '.' )[0]] = module;
+      });
+      return acc;
+    }, {} as Record<string, string> );
+
+    const serviceSubscriptions = !consumes ? {} : reduce(( subscriptions, events, eventHandler: string ) => {
+      const kebabHandler = paramCase( eventHandler );
+      const subscription = reduce(( subscription, event ) => {
+        const producer = eventProducerMap[event];
+        if ( producer ) {
+          subscription.events.push( event );
+          subscription.producers.add( producer.system );
+        }
+        return subscription;
+      }, { events: [], producers: new Set() } as { events: string[], producers: Set<string>}, events );
+
+      const id = `${subscription.producers.size > 1 ? 'multi' : Array.from( subscription.producers )[0]}-${eventHandler}`;
+      subscriptions[id] = {
+        consumer: eventHandler,
+        module: artifactModuleMap[eventHandler],
+        producers: Array.from( subscription.producers ),
+        endpoint: `/${kebabHandler}`,
+        events: subscription.events,
+      };
+      return subscriptions;
+    }, {} as Record<string, Partial<StructureSubscriptionType>>, consumes );
+    return {
+      subscriptions: {
+        ...maps.subscriptions, ...serviceSubscriptions
+      },
+      servicesSubscriptions: { ...maps.servicesSubscriptions, [service]: Object.keys( serviceSubscriptions ) }
+    };
+  } ,{ subscriptions:{}, servicesSubscriptions: {}} as DiscoveredSubscriptions , services ) as DiscoveredSubscriptions;
+};
+
+export const getThisServiceIps = () => {
+  const interfaces = os.networkInterfaces();
+  return Object.values( interfaces ).reduce(( ips, interfaceInfo ) => {
+    if ( interfaceInfo )
+      return ips.concat( interfaceInfo.filter( info => info.family === 'IPv4' && info.internal === false ).map( info => info.address ));
+    else return ips;
+  }, [] as string[]);
+};

package/src/ports/logger/index.ts

@@ -0,0 +1,34 @@
+import { LevelLogMap, LoggerAdapter, LoggerConfig, LoggerPort, LoggerType, LogLevelMap } from '@bett3r-dev/pv3-types';
+
+export const Logger = ( config: LoggerConfig, ...adapters: LoggerAdapter[]): LoggerPort => {
+  const skipModule = ( module: string ) =>
+    config?.modulesBlacklist?.includes( module ) || ( config?.modulesWhitelist && !config?.modulesWhitelist?.includes( module ));
+
+  function createLoggerInstance( module: string ): LoggerType {
+    const LogLevel = LogLevelMap[config.logLevel || 'info'];
+
+    const internalLog = skipModule( module ) ?
+      () => () => {}
+      : ( level: number ) => ( ...args: any[]) => {
+        if ( LogLevel >= level )
+          adapters.forEach( a=> {
+            a.log( args, LevelLogMap[level], module );
+          });
+      };
+
+    const logger: LoggerType = {
+      error: internalLog( 0 ),
+      warn: internalLog( 1 ),
+      info: internalLog( 2 ),
+      log: internalLog( 3 ),
+      debug: internalLog( 4 ),
+      silly: internalLog( 5 ),
+    };
+    return logger;
+  }
+
+  return {
+    createLoggerInstance
+  };
+};
+