@bett3r-dev/pv3 1.0.0 → 2.5.1

package/src/ports/infrastructure/__tests__/fixture/structureState.ts

@@ -1,254 +0,0 @@
-export const structureState = {
-  state: {
-    brokers: {
-      '10.0.0.13': {
-        subscriptions: {
-          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
-            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-0': {
-              host: '10.0.0.13',
-              index: 0,
-            },
-          },
-          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
-            'ArithmeticOperationsAggregate-TotalByColorReadModel-0': {
-              host: '10.0.0.13',
-              index: 0,
-            },
-          },
-        },
-      },
-      '10.0.0.14': {
-        subscriptions: {
-          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
-            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-1': {
-              host: '10.0.0.14',
-              index: 1,
-            },
-          },
-          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
-            'ArithmeticOperationsAggregate-TotalByColorReadModel-1': {
-              host: '10.0.0.14',
-              index: 1,
-            },
-          },
-        },
-      },
-      '10.0.0.15': {
-        subscriptions: {
-          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
-            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-2': {
-              host: '10.0.0.15',
-              index: 2,
-            },
-          },
-          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
-            'ArithmeticOperationsAggregate-TotalByColorReadModel-2': {
-              host: '10.0.0.15',
-              index: 2,
-            },
-          },
-        },
-      },
-      '10.0.0.16': {
-        subscriptions: {
-          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
-            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-3': {
-              host: '10.0.0.16',
-              index: 3,
-            },
-          },
-          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
-            'ArithmeticOperationsAggregate-TotalByColorReadModel-3': {
-              host: '10.0.0.16',
-              index: 3,
-            },
-          },
-        },
-      },
-      '10.0.0.18': {
-        subscriptions: {
-          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
-            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-4': {
-              host: '10.0.0.18',
-              index: 4,
-            },
-          },
-          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
-            'ArithmeticOperationsAggregate-TotalByColorReadModel-4': {
-              host: '10.0.0.18',
-              index: 4,
-            },
-          },
-        },
-      },
-      '10.0.0.19': {
-        subscriptions: {
-          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
-            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-5': {
-              host: '10.0.0.19',
-              index: 5,
-            },
-          },
-          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
-            'ArithmeticOperationsAggregate-TotalByColorReadModel-5': {
-              host: '10.0.0.19',
-              index: 5,
-            },
-          },
-        },
-      },
-      '10.0.0.20': {
-        subscriptions: {
-          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
-            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-6': {
-              host: '10.0.0.20',
-              index: 6,
-            },
-          },
-          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
-            'ArithmeticOperationsAggregate-TotalByColorReadModel-6': {
-              host: '10.0.0.20',
-              index: 6,
-            },
-          },
-        },
-      },
-      '10.0.0.21': {
-        subscriptions: {
-          'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
-            'ArithmeticOperationsAggregate-OperationCountByColorReadModel-7': {
-              host: '10.0.0.21',
-              index: 7,
-            },
-          },
-          'ArithmeticOperationsAggregate-TotalByColorReadModel': {
-            'ArithmeticOperationsAggregate-TotalByColorReadModel-7': {
-              host: '10.0.0.21',
-              index: 7,
-            },
-          },
-        },
-      },
-    },
-    services: {
-      service0: {
-        port: 2013,
-        consumes: {
-          OperationCountByColorReadModel: [
-            'DividedBy_17',
-            'MultipliedBy_18',
-            'MultipliedBy_2',
-            'MultipliedBy_5',
-            'Subtracted_50',
-            'Added_20',
-            'Added_7',
-            'Added_3',
-            'DividedBy_3',
-          ],
-          TotalByColorReadModel: [
-            'MultipliedBy_18',
-            'DividedBy_17',
-            'MultipliedBy_5',
-            'MultipliedBy_2',
-            'Subtracted_50',
-            'Added_20',
-            'Added_7',
-            'Added_3',
-            'DividedBy_3',
-          ],
-        },
-        produces: {
-          ArithmeticOperationsAggregate: [
-            'DividedBy_17',
-            'Added_20',
-            'MultipliedBy_5',
-            'Added_3',
-            'Added_7',
-            'MultipliedBy_2',
-            'Subtracted_50',
-            'DividedBy_3',
-            'MultipliedBy_18',
-          ],
-        },
-        isBroker: true,
-        isService: true,
-        endpoints: [
-          'GET:/arithmetic-operations-aggregate/{id}',
-          'GET:/arithmetic-operations-aggregate/{id}/stream',
-          'POST:/arithmetic-operations-aggregate/{id}/divide-by-17',
-          'POST:/arithmetic-operations-aggregate/{id}/add-20',
-          'POST:/arithmetic-operations-aggregate/{id}/multiply-by-5',
-          'POST:/arithmetic-operations-aggregate/{id}/add-3',
-          'POST:/arithmetic-operations-aggregate/{id}/add-7',
-          'POST:/arithmetic-operations-aggregate/{id}/multiply-by-2',
-          'POST:/arithmetic-operations-aggregate/{id}/subtract-50',
-          'POST:/arithmetic-operations-aggregate/{id}/divide-by-3',
-          'POST:/arithmetic-operations-aggregate/{id}/multiply-by-18',
-          'POST:/operation-count-by-color-read-model',
-          'GET:/operation-count-by-color-read-model/{id}?',
-          'POST:/total-by-color-read-model',
-          'GET:/total-by-color-read-model/{id}?',
-        ],
-        possibleSubscriptions: [
-          'ArithmeticOperationsAggregate-OperationCountByColorReadModel',
-          'ArithmeticOperationsAggregate-TotalByColorReadModel',
-        ],
-        hosts: [
-          '10.0.0.13',
-          '10.0.0.14',
-          '10.0.0.15',
-          '10.0.0.16',
-          '10.0.0.18',
-          '10.0.0.19',
-          '10.0.0.20',
-          '10.0.0.21',
-        ],
-      },
-    },
-    subscriptions: {
-      'ArithmeticOperationsAggregate-OperationCountByColorReadModel': {
-        consumer: 'OperationCountByColorReadModel',
-        endpoint: '/operation-count-by-color-read-model',
-        events: [
-          'DividedBy_17',
-          'MultipliedBy_18',
-          'MultipliedBy_2',
-          'MultipliedBy_5',
-          'Subtracted_50',
-          'Added_20',
-          'Added_7',
-          'Added_3',
-          'DividedBy_3',
-        ],
-        producer: 'ArithmeticOperationsAggregate',
-        dataSplits: {
-          manual: [],
-          custom: [],
-          automatic: [],
-        },
-      },
-      'ArithmeticOperationsAggregate-TotalByColorReadModel': {
-        consumer: 'OperationCountByColorReadModel',
-        endpoint: '/total-by-color-read-model',
-        events: [
-          'MultipliedBy_18',
-          'DividedBy_17',
-          'MultipliedBy_5',
-          'MultipliedBy_2',
-          'Subtracted_50',
-          'Added_20',
-          'Added_7',
-          'Added_3',
-          'DividedBy_3',
-        ],
-        producer: 'ArithmeticOperationsAggregate',
-        dataSplits: {
-          manual: [],
-          custom: [],
-          automatic: [],
-        },
-      },
-    },
-  },
-  expectedVersion: 8,
-};

package/src/ports/infrastructure/__tests__/infrastructure.test.ts

@@ -1,11 +0,0 @@
-import { getStructureDifferences } from '..';
-import { infrastructureServices, structureState } from './fixture';
-
-describe( 'Infrastructure Port', () => {
-  describe( 'getStructureDifferences', () => {
-    it( 'works', async () => {
-      // const result = getStructureDifferences( structureState, infrastructureServices );
-      // result;//?
-    });
-  });
-});

package/src/ports/infrastructure/index.ts

@@ -1,162 +0,0 @@
-import { reduce } from '@bett3r-dev/bett3r-utils';
-import { EnvironmentServiceType, InfrastructureAdapterType, InfrastructureConfigType, InfrastructurePortType, InfrastructureServiceType, InfrastructureThisServiceType, Ports, StructureServiceType } from '@bett3r-dev/pv3-types';
-import { OpenAPIV3_1 } from 'openapi-types';
-import { append, assoc, compose, converge, identity, omit } from 'ramda';
-import { LocalInfrastructure } from '../../adapters/infrastructure';
-import { ClusterStructureType, StructureState, UpsertHostCommandType } from '@bett3r-dev/pv3-types';
-import { createPort } from '../base';
-import { discoverSubscriptions, getServiceDefinition, getThisServiceIps } from './utils';
-export { getThisServiceIps } from './utils';
-
-export const getStructureDifferences = ( structureState: StructureState, infra: InfrastructureServiceType[]) => {
-  const serviceHostsSet = new Set( Object.values( structureState.state.services ).reduce(( hosts, service ) => hosts.concat( service.hosts ), [] as string[]));
-  const upsertHosts: UpsertHostCommandType[] = [];
-  const infraServices = infra.reduce(( services, service ) => {
-    if ( service.isService )
-      services[service.service] = service as any;
-    return services;
-  }, {} as Record<string, Omit<StructureServiceType, 'hosts'>> );
-  const { subscriptions, servicesSubscriptions } = discoverSubscriptions( infraServices );
-  for ( const service of infra ) {
-    for ( const { host } of service.instances ) {
-      serviceHostsSet.delete( host );
-      upsertHosts.push({
-        ...omit(['instances'], service ),
-        host,
-        possibleSubscriptions: servicesSubscriptions[service.service]
-      });
-    }
-  }
-  const removeHosts = Array.from( serviceHostsSet );
-  return {
-    upsertHosts,
-    removeHosts,
-    subscriptions
-  };
-};
-
-const checkInfrastructureIntegrity = ( structure: ClusterStructureType ) => async ( infra: InfrastructureServiceType[], retryBackoff = 100 ) => {
-  const structureState = await structure.getCurrentState();
-  const differences = getStructureDifferences( structureState, infra );
-  console.log( '$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ STRUCTURE DIFFERENCE Version', structureState.version, differences.upsertHosts.map( x => x.host ));
-  await structure.SetStructure( differences, structureState.version )
-    .catch(( e ) => {
-      if ( e.type !== 'ConcurrencyError' )
-        throw e;
-      else {
-        console.log( '$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ CONCURRENCY ERROR', e );
-        return new Promise(( resolve ) => {
-          setTimeout(() => {
-            checkInfrastructureIntegrity( structure )( infra, retryBackoff * 2 )
-              .then( resolve );
-          }, retryBackoff );
-        });
-      }
-    });
-  return infra;
-};
-
-const getCurrentServiceInfo = ( services: InfrastructureServiceType[]): InfrastructureThisServiceType => {
-  const ips = getThisServiceIps();
-
-  const brokerHosts = services
-    .reduce(( brokers, service, index ) =>{
-      if ( !service.isBroker ) return brokers;
-      brokers.push( ...service.instances.map( assoc( 'serviceIndex', index )));
-      return brokers;
-    }, [] as {host: string, timeRunning: number, serviceIndex:number}[])
-    .sort(( a, b ) => a.timeRunning - b.timeRunning );
-
-  const hostIndex = brokerHosts
-    .findIndex( host => ips.includes( host.host ));
-  const brokerHost = brokerHosts[hostIndex];
-
-  if ( hostIndex === -1 ) throw new Error( 'Service not found in infrastructure' );
-  return {
-    ...services[brokerHost.serviceIndex],
-    host: brokerHost.host,
-    leadershipRank: hostIndex,
-  };
-};
-
-export const getInfrastructureServicesDefinition = async ( infraServices: EnvironmentServiceType[]): Promise<InfrastructureServiceType[]> => {
-  return infraServices.map(( service ) => {
-    return {
-      service: service.service,
-      port: service.port,
-      instances: service.instances,
-      openApiEndpoint: service.openApiEndpoint,
-      ...( service.openApi ? getServiceDefinition( service.openApi ) : {})
-    } as InfrastructureServiceType;
-  });
-};
-
-
-export const InfrastructurePort = ( ports: Pick<Ports, 'logger'|'endpoints'|'configuration'>, config: InfrastructureConfigType, structure: ClusterStructureType, adapter?: InfrastructureAdapterType ): InfrastructurePortType => {
-  let servicesOpenApiDocumentation: Record<string, OpenAPIV3_1.Document> = {};
-  let _adapter: InfrastructureAdapterType;
-  let thisService: InfrastructureThisServiceType;
-  let adapters: InfrastructureAdapterType[] = append( LocalInfrastructure( ports ), adapter ? [adapter] : []);
-
-  const assignOpenApiDocumentation = compose(
-    ( services: any ) => servicesOpenApiDocumentation = services,
-    reduce(( docs, serviceDefinition ) => assoc( serviceDefinition.service, serviceDefinition.openApi, docs ), {})
-  );
-
-  const setOpenApiDocs = converge(( x, _ ) => x, [ identity, assignOpenApiDocumentation ]);
-
-  const start = async () => {
-    const _adapters = await Promise.allSettled( adapters!.map( adapter => adapter.start()));
-    const index = _adapters.findIndex( adapter => adapter.status === 'fulfilled' );
-    _adapter = adapters![index];
-    const checkInfrastructure = checkInfrastructureIntegrity( structure );
-    const infraServices = await _adapter.getServices().then( setOpenApiDocs ) as EnvironmentServiceType[];
-    const infraServicesDefinition = await getInfrastructureServicesDefinition( infraServices );
-    thisService = getCurrentServiceInfo( infraServicesDefinition );
-
-    _adapter.listenForInfrastructureChanges(( action, info ) => {
-      console.log( 'LOS SERVICES QUE RECIBI --------- ', info.instances.map( i=>i.host ));
-      if ( action === 'create' && info.openApi.tags?.find( tag => tag.name === 'pv3.outbox' )){
-        return;
-      }
-      //TODO: Crear una mejor manera de manejar el exponential backoff. y operaciones idempotentes
-      setTimeout(() => {
-        _adapter.getServices()
-          // .then(( s ) => {
-          //   const service = s.find( s=> s.service === info.service );
-          //   if ( service?.openApi.tags?.find( tag => tag.name === 'pv3.outbox' )){
-          //     return Promise.reject( 'IgnoringOutboxService' );
-          //   }
-          //   // return s.filter(( s ) => {
-          //   //   info.service === s.service
-          //   //   return s.openApi.tags?.find(( tag ) => tag.name !== 'pv3.outbox' );
-          //   // });
-          // })
-          .then( x=> { console.log( 'LOS SERVICES QUE VOY A AGREGAR --------- ', x.map( s=>s.instances.map( i=>i.host ))); return x; })
-          .then<EnvironmentServiceType[]>( setOpenApiDocs )
-          .then( getInfrastructureServicesDefinition )
-          .then( checkInfrastructure );
-      }, ( config.infrastructureLeaderRandomBackoff || 1000 ) * ( thisService.leadershipRank || 0 ));
-    });
-
-    await checkInfrastructure( infraServicesDefinition );
-    environments.isStarted = true;
-    environments.emitter.emit( 'started' );
-  };
-
-  const environments = createPort({
-    _adapter: () => _adapter,
-    name: 'Infrastructure',
-    getServicesOpenApiDocumentation: () => servicesOpenApiDocumentation,
-    getThisServiceInfo: () => thisService,
-    getServices: () => _adapter!.getServices(),
-    scaleService: ( service: string, replicasCount: number ) => _adapter!.scaleService( service, replicasCount ),
-    listenForInfrastructureChanges: ( callback: ( event: 'destroy' | 'create', service: EnvironmentServiceType ) => void ) => _adapter!.listenForInfrastructureChanges( callback ),
-    reloadLoadBalancer: () => _adapter!.reloadLoadBalancer(),
-    start,
-    stop: async () => _adapter.stop()
-  });
-
-  return environments;
-};
-

package/src/ports/infrastructure/utils.ts

@@ -1,130 +0,0 @@
-import { reduce } from '@bett3r-dev/bett3r-utils';
-import { StructureServiceType, StructureSubscriptionType } from '@bett3r-dev/pv3-types';
-import { paramCase } from 'change-case';
-import { OpenAPIV3_1 } from 'openapi-types';
-import os from 'os';
-import { omit } from 'ramda';
-
-export type DiscoveredSubscriptions = {
-  subscriptions: {
-    [id: string]: StructureSubscriptionType;
-  };
-  servicesSubscriptions: {
-    [host: string]: string[]
-  }
-}
-
-const extractPathEndpoints = ( path: OpenAPIV3_1.PathItemObject, url: string ) => {
-  return [ 'post', 'get', 'put', 'delete', 'patch', 'head', 'options' ]
-    .reduce(( endpoints, method ) => {
-      if ( path[method]) endpoints.push( `${method.toUpperCase()}:${url}` );
-      return endpoints;
-    }, [] as string[]);
-};
-
-
-export const getServiceDefinition = ( swagger: OpenAPIV3_1.Document ): StructureServiceType => {
-  const producerTags = [ 'aggregate', 'system' ];
-  const postSchemaProperty = 'responses';
-  const definition = Object.keys( swagger?.paths as OpenAPIV3_1.PathsObject || {}).reduce(( definition, url )=>{
-    const path = swagger.paths![url];
-    if ( path?.post?.tags?.some( t=> producerTags.includes( t ))){
-      const postSchemaBase = ( path.post as any )[postSchemaProperty];
-      const postSchema = postSchemaProperty === 'responses' ? postSchemaBase[200] : postSchemaBase;
-      const schema = postSchema.content['application/json'].schema;
-      let events;
-      if ( schema.anyOf ){
-        events = schema.anyOf
-          .map( s=>{
-            const ref = s.properties.data;
-            return ref.$ref.slice( ref.$ref.lastIndexOf( '/' ) + 1 );
-          });
-      } else if ( schema.properties?.events?.properties?.data?.anyOf ) {
-        events = schema.properties?.events?.properties?.data?.anyOf
-          .map( ref=>{
-            return ref.$ref.slice( ref.$ref.lastIndexOf( '/' ) + 1 );
-          });
-      } else {
-        events = [];
-      }
-      const handler = path.post?.tags?.filter( t=>!producerTags.includes( t ))[0];
-      definition.produces![handler] = [ ...( definition.produces![handler] || []), ...events ];
-    } else if ( path?.post?.tags?.some( t=> [ 'policy', 'readmodel' ].includes( t ))){
-      const events = ( path.post.requestBody as any ).content['application/json'].schema.anyOf.map( s=>{
-        const ref = s.properties.data;
-        return ref.$ref.slice( ref.$ref.lastIndexOf( '/' ) + 1 );
-      });
-      const handler = path.post?.tags?.filter( t=>![ 'policy', 'readmodel' ].includes( t ))[0];
-      definition.consumes![handler] = events;
-    }
-    return {
-      ...definition,
-      endpoints: [ ...definition.endpoints, ...extractPathEndpoints( path!, url ) ]
-    };
-  }, { consumes: {}, produces: {}, endpoints: []});
-  return {
-    ...omit(['endpoints'], definition ),
-    isBroker: swagger.tags?.some( t=>t.name === 'pv3.outbox' ),
-    isService: swagger.tags?.some( t=>t.name === 'pv3.service' ),
-    possibleModules: JSON.parse(( swagger.tags?.find( t=>t.name.startsWith( 'modules' )) || { name:'modules={}' }).name.split( 'modules=' )[1]),
-    endpoints: definition.endpoints
-  } as StructureServiceType;
-};
-
-export const discoverSubscriptions = ( services: Record<string, Omit<StructureServiceType, 'hosts'>> ): DiscoveredSubscriptions => {
-  const eventProducerMap = reduce(( eventMap, { produces }, service ) => {
-    const serviceMap = reduce(( eventMap, events, system: string ) => {
-      events.forEach( e=>{
-        eventMap[e] = { system, service: service };
-      });
-      return eventMap;
-    }, {} as Record<string, {system: string, service: string}>, produces );
-    return { ...eventMap, ...serviceMap };
-  } ,{} as Record<string, {system: string, service: string}>,services );
-
-  return reduce(( maps, { consumes, possibleModules }, service ) => {
-    const artifactModuleMap = Object.entries( possibleModules || {}).reduce(( acc, [ module, artifacts ]) => {
-      artifacts.forEach(( artifact ) => {
-        acc[artifact.split( '.' )[0]] = module;
-      });
-      return acc;
-    }, {} as Record<string, string> );
-
-    const serviceSubscriptions = !consumes ? {} : reduce(( subscriptions, events, eventHandler: string ) => {
-      const kebabHandler = paramCase( eventHandler );
-      const subscription = reduce(( subscription, event ) => {
-        const producer = eventProducerMap[event];
-        if ( producer ) {
-          subscription.events.push( event );
-          subscription.producers.add( producer.system );
-        }
-        return subscription;
-      }, { events: [], producers: new Set() } as { events: string[], producers: Set<string>}, events );
-
-      const id = `${subscription.producers.size > 1 ? 'multi' : Array.from( subscription.producers )[0]}-${eventHandler}`;
-      subscriptions[id] = {
-        consumer: eventHandler,
-        module: artifactModuleMap[eventHandler],
-        producers: Array.from( subscription.producers ),
-        endpoint: `/${kebabHandler}`,
-        events: subscription.events,
-      };
-      return subscriptions;
-    }, {} as Record<string, Partial<StructureSubscriptionType>>, consumes );
-    return {
-      subscriptions: {
-        ...maps.subscriptions, ...serviceSubscriptions
-      },
-      servicesSubscriptions: { ...maps.servicesSubscriptions, [service]: Object.keys( serviceSubscriptions ) }
-    };
-  } ,{ subscriptions:{}, servicesSubscriptions: {}} as DiscoveredSubscriptions , services ) as DiscoveredSubscriptions;
-};
-
-export const getThisServiceIps = () => {
-  const interfaces = os.networkInterfaces();
-  return Object.values( interfaces ).reduce(( ips, interfaceInfo ) => {
-    if ( interfaceInfo )
-      return ips.concat( interfaceInfo.filter( info => info.family === 'IPv4' && info.internal === false ).map( info => info.address ));
-    else return ips;
-  }, [] as string[]);
-};

package/src/ports/logger/index.ts

@@ -1,34 +0,0 @@
-import { LevelLogMap, LoggerAdapter, LoggerConfig, LoggerPort, LoggerType, LogLevelMap } from '@bett3r-dev/pv3-types';
-
-export const Logger = ( config: LoggerConfig, ...adapters: LoggerAdapter[]): LoggerPort => {
-  const skipModule = ( module: string ) =>
-    config?.modulesBlacklist?.includes( module ) || ( config?.modulesWhitelist && !config?.modulesWhitelist?.includes( module ));
-
-  function createLoggerInstance( module: string ): LoggerType {
-    const LogLevel = LogLevelMap[config.logLevel || 'info'];
-
-    const internalLog = skipModule( module ) ?
-      () => () => {}
-      : ( level: number ) => ( ...args: any[]) => {
-        if ( LogLevel >= level )
-          adapters.forEach( a=> {
-            a.log( args, LevelLogMap[level], module );
-          });
-      };
-
-    const logger: LoggerType = {
-      error: internalLog( 0 ),
-      warn: internalLog( 1 ),
-      info: internalLog( 2 ),
-      log: internalLog( 3 ),
-      debug: internalLog( 4 ),
-      silly: internalLog( 5 ),
-    };
-    return logger;
-  }
-
-  return {
-    createLoggerInstance
-  };
-};
-