@flowcore/cli 2.8.2 → 2.9.0

package/CHANGELOG.md

@@ -10,6 +10,14 @@
 
 * added description to start that includes week ([58687a7](https://github.com/flowcore-io/flowcore-cli/commit/58687a7bbb66aaa5d6da26af88e555cbb1e72467))
 
+## [2.9.0](https://github.com/flowcore-io/flowcore-cli/compare/v2.8.2...v2.9.0) (2024-03-07)
+
+
+### Features
+
+* add support for sending only the event payload to the destination in processEvents function ([fa44f87](https://github.com/flowcore-io/flowcore-cli/commit/fa44f8742c565df5b7482c21e7a2f0b597e671e0))
+* enhance stream command with max event option ([5bb9ff2](https://github.com/flowcore-io/flowcore-cli/commit/5bb9ff26c6d59007442cab1c1bcb2edae7182858))
+
 ## [2.8.2](https://github.com/flowcore-io/flowcore-cli/compare/v2.8.1...v2.8.2) (2024-02-28)
 
 

package/README.md

@@ -18,7 +18,7 @@ $ npm install -g @flowcore/cli
 $ flowcore COMMAND
 running command...
 $ flowcore (--version)
-@flowcore/cli/2.8.2 linux-x64 node-v20.11.1
+@flowcore/cli/2.9.0 linux-x64 node-v20.11.1
 $ flowcore --help [COMMAND]
 USAGE
   $ flowcore COMMAND
@@ -75,7 +75,7 @@ EXAMPLES
   $ flowcore apply -f ./path/to/manifest.yml
 ```
 
-_See code: [src/commands/apply.ts](https://github.com/flowcore-io/flowcore-cli/blob/v2.8.2/src/commands/apply.ts)_
+_See code: [src/commands/apply.ts](https://github.com/flowcore-io/flowcore-cli/blob/v2.9.0/src/commands/apply.ts)_
 
 ## `flowcore autocomplete [SHELL]`
 
@@ -722,8 +722,8 @@ Stream events from a datacore running on the Flowcore Platform
 
 ```
 USAGE
-  $ flowcore stream STREAM [--profile <value>] [-d <value>] [-H <value>] [-j] [-l] [-o http|log] [-c] [-s
-    <value>] [-t <value>]
+  $ flowcore stream STREAM [--profile <value>] [-d <value>] [-H <value>] [-j] [-l] [-m <value>] [-o
+    http|log] [-p] [-c] [-s <value>] [-t <value>]
 
 ARGUMENTS
   STREAM  stream url to connect to
@@ -734,8 +734,10 @@ FLAGS
   -d, --destination=<value>  [default: http://localhost:3000/transform] Destination to send events to
   -j, --json                 Output json only
   -l, --[no-]live            Change to live mode when reaching last time bucket
+  -m, --max=<value>          Maximum number of events to send to the destination
   -o, --output=<option>      [default: http] Output format
                              <options: http|log>
+  -p, --payload              Only send the event payload to the destination
   -s, --start=<value>        Start time bucket to stream from, example: (1y, 1m, 1w, 1d, 1h, now)
   -t, --timeout=<value>      [default: 5000] Timeout in milliseconds to wait for a response from the destination
       --profile=<value>      Specify the configuration profile to use
@@ -753,7 +755,7 @@ EXAMPLES
   $ flowcore stream https://flowcore.io/<org>/<data core>/<flow type>/[<event type1>,<event type2>,<event type3>].stream -o log -s 3m
 ```
 
-_See code: [src/commands/stream.ts](https://github.com/flowcore-io/flowcore-cli/blob/v2.8.2/src/commands/stream.ts)_
+_See code: [src/commands/stream.ts](https://github.com/flowcore-io/flowcore-cli/blob/v2.9.0/src/commands/stream.ts)_
 
 ## `flowcore version`
 

package/dist/commands/stream.d.ts

@@ -10,7 +10,9 @@ export default class Stream extends BaseCommand<typeof Stream> {
         header: import("@oclif/core/lib/interfaces/parser.js").OptionFlag<string[], import("@oclif/core/lib/interfaces/parser.js").CustomOptions>;
         json: import("@oclif/core/lib/interfaces/parser.js").BooleanFlag<boolean>;
         live: import("@oclif/core/lib/interfaces/parser.js").BooleanFlag<boolean>;
+        max: import("@oclif/core/lib/interfaces/parser.js").OptionFlag<number, import("@oclif/core/lib/interfaces/parser.js").CustomOptions>;
         output: import("@oclif/core/lib/interfaces/parser.js").OptionFlag<string, import("@oclif/core/lib/interfaces/parser.js").CustomOptions>;
+        payload: import("@oclif/core/lib/interfaces/parser.js").BooleanFlag<boolean>;
         scan: import("@oclif/core/lib/interfaces/parser.js").BooleanFlag<boolean>;
         start: import("@oclif/core/lib/interfaces/parser.js").OptionFlag<string | undefined, import("@oclif/core/lib/interfaces/parser.js").CustomOptions>;
         timeout: import("@oclif/core/lib/interfaces/parser.js").OptionFlag<number, import("@oclif/core/lib/interfaces/parser.js").CustomOptions>;

package/dist/commands/stream.js

@@ -31,7 +31,9 @@ export default class Stream extends BaseCommand {
         header: Flags.string({ char: 'H', default: [], description: 'header to send with the request, example: (-H \'Authorization: Bearer TOKEN\')', multiple: true }),
         json: Flags.boolean({ char: 'j', description: 'Output json only' }),
         live: Flags.boolean({ allowNo: true, char: 'l', default: true, description: 'Change to live mode when reaching last time bucket' }),
+        max: Flags.integer({ char: 'm', default: 0, description: 'Maximum number of events to send to the destination' }),
         output: Flags.string({ char: 'o', default: 'http', description: 'Output format', options: ['http', 'log'] }),
+        payload: Flags.boolean({ char: 'p', default: false, description: 'Only send the event payload to the destination' }),
         scan: Flags.boolean({ char: 'c', default: false, description: 'Scan the full time range' }),
         start: Flags.string({ char: 's', description: 'Start time bucket to stream from, example: (1y, 1m, 1w, 1d, 1h, now)' }),
         timeout: Flags.integer({ char: 't', default: 5000, description: 'Timeout in milliseconds to wait for a response from the destination' }),
@@ -113,8 +115,8 @@ export default class Stream extends BaseCommand {
         const observer = new Subject();
         !this.flags.json && this.log(ux.colorize("blackBright", `Starting to stream events for ${dataCoreId} - ${aggregator} - ${eventTypes}, press ${ux.colorize("whiteBright", "ctrl+c")} to stop`));
         // eslint-disable-next-line no-void
-        void this.streamEvents(dataCoreId, aggregator, eventTypes, firstTimeBucket, lastTimeBucket, graphqlClient, observer, flags.live, afterEventId);
-        await this.processEvents(observer, flags.destination, flags.output, flags.header);
+        void this.streamEvents(dataCoreId, aggregator, eventTypes, firstTimeBucket, lastTimeBucket, graphqlClient, observer, flags.live, afterEventId, flags.max);
+        await this.processEvents(observer, flags.destination, flags.output, flags.header, flags.payload);
     }
     calculateBuckets(timediff, startTimeBucket) {
         return _.times(timediff + 1, (n) => dayjs(startTimeBucket).add(n, "hour").format(TIME_BUCKET_PATTERN));
@@ -165,7 +167,7 @@ export default class Stream extends BaseCommand {
         }
         return timeBuckets;
     }
-    async processEvents(observer, destination, output, header = []) {
+    async processEvents(observer, destination, output, header = [], payload = false) {
         let done = false;
         let events = [];
         const warn = this.flags.json ? () => { } : this.warn.bind(this);
@@ -188,11 +190,12 @@ export default class Stream extends BaseCommand {
             events = [];
             while (!processingQueue.isEmpty()) {
                 const event = processingQueue.dequeue();
+                const outputPayload = payload ? event.payload : event;
                 if (output === "log") {
                     this.flags.json ? this.log(JSON.stringify({
-                        event,
+                        event: outputPayload,
                         type: "input",
-                    }, null, 2)) : this.log(JSON.stringify(event));
+                    }, null, 2)) : this.log(JSON.stringify(outputPayload));
                     continue;
                 }
                 !this.flags.json && this.log(`Sending event to ${destination}: ${event.eventId}`);
@@ -204,7 +207,7 @@ export default class Stream extends BaseCommand {
                     }
                     // eslint-disable-next-line no-await-in-loop
                     const result = await Promise.race([fetch(destination, {
-                            body: JSON.stringify(event),
+                            body: JSON.stringify(outputPayload),
                             headers: {
                                 "Content-Type": "application/json",
                                 ...headers,
@@ -260,7 +263,7 @@ export default class Stream extends BaseCommand {
             }
         }
     }
-    async streamEvents(dataCoreId, aggregator, eventTypes, firstTimeBucket, lastTimeBucket, graphqlClient, observer, live, afterEventId) {
+    async streamEvents(dataCoreId, aggregator, eventTypes, firstTimeBucket, lastTimeBucket, graphqlClient, observer, live, afterEventId, maxEvents = 0) {
         const startTimeBucket = dayjs(firstTimeBucket, TIME_BUCKET_HOUR_PATTERN);
         const currentTime = () => dayjs.utc();
         let liveMode = false;
@@ -268,6 +271,7 @@ export default class Stream extends BaseCommand {
         const timediff = this.calculateTimeDifferenceInHours(lastTimeBucket, startTimeBucket, firstTimeBucket);
         let timeBuckets = [];
         timeBuckets = this.flags.scan ? this.calculateBuckets(timediff, startTimeBucket) : (await this.fetchTimeBuckets(dataCoreId, aggregator, eventTypes, startTimeBucket, lastTimeBucket, graphqlClient));
+        let eventsCount = 0;
         // eslint-disable-next-line no-constant-condition
         while (true) {
             const timeBucket = timeBuckets.shift();
@@ -294,12 +298,23 @@ export default class Stream extends BaseCommand {
                         dataCoreId,
                         eventTypes,
                         timeBucket,
+                        ...(maxEvents > 0 ? { pageSize: maxEvents - eventsCount } : {})
                     });
                     cursor = response.datacore.fetchEvents.cursor;
                     for (const event of response.datacore.fetchEvents.events) {
                         found = true;
                         observer.next(event);
                         lastEventId = event.eventId;
+                        eventsCount++;
+                        // eslint-disable-next-line max-depth
+                        if (maxEvents > 0 && eventsCount >= maxEvents) {
+                            observer.complete();
+                            // eslint-disable-next-line max-depth
+                            if (!this.flags.json) {
+                                ux.action.stop(`Fetched total event count of ${eventsCount}, stopping`);
+                            }
+                            return;
+                        }
                     }
                 }
                 catch (error) {

package/oclif.manifest.json

@@ -103,6 +103,15 @@
           "allowNo": true,
           "type": "boolean"
         },
+        "max": {
+          "char": "m",
+          "description": "Maximum number of events to send to the destination",
+          "name": "max",
+          "default": 0,
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
         "output": {
           "char": "o",
           "description": "Output format",
@@ -116,6 +125,13 @@
           ],
           "type": "option"
         },
+        "payload": {
+          "char": "p",
+          "description": "Only send the event payload to the destination",
+          "name": "payload",
+          "allowNo": false,
+          "type": "boolean"
+        },
         "scan": {
           "char": "c",
           "description": "Scan the full time range",
@@ -157,5 +173,5 @@
       ]
     }
   },
-  "version": "2.8.2"
+  "version": "2.9.0"
 }

package/package.json

@@ -90,7 +90,7 @@
     "prestart": "npm run build",
     "update-schema": "rover graph introspect https://graph.api.staging.flowcore.io/graphql -o schema.gql"
   },
-  "version": "2.8.2",
+  "version": "2.9.0",
   "bugs": "https://github.com/flowcore-io/flowcore-cli/issues",
   "keywords": [
     "flowcore",