@flowcore/cli 1.3.0 → 2.0.1

package/CHANGELOG.md

@@ -1,23 +1,32 @@
-## [0.4.9](https://github.com/oclif/hello-world/compare/0.4.8...0.4.9) (2023-11-26)
+## [1.3.0](https://github.com/flowcore-io/flowcore-cli/compare/v1.2.0...v1.3.0) (2024-01-09)
+
+
+### Features
+
+* added timeout for destination as flag ([f731aac](https://github.com/flowcore-io/flowcore-cli/commit/f731aac025262179af1fd56e2ab2477a674635f9))
 
 
 ### Bug Fixes
 
-* **deps:** bump @oclif/plugin-plugins from 4.1.5 to 4.1.8 ([eb58fcf](https://github.com/oclif/hello-world/commit/eb58fcf7f6535189dc1acbec3628ed37702df7e5))
+* added description to start that includes week ([58687a7](https://github.com/flowcore-io/flowcore-cli/commit/58687a7bbb66aaa5d6da26af88e555cbb1e72467))
 
+## [2.0.1](https://github.com/flowcore-io/flowcore-cli/compare/v2.0.0...v2.0.1) (2024-01-15)
 
 
-## [1.3.0](https://github.com/flowcore-io/flowcore-cli/compare/v1.2.0...v1.3.0) (2024-01-09)
+### Bug Fixes
 
+* set to use scan mode when no time buckets are found ([f1fac44](https://github.com/flowcore-io/flowcore-cli/commit/f1fac447a058eb7a61eeff5690208203d27c0a79))
 
-### Features
+## [2.0.0](https://github.com/flowcore-io/flowcore-cli/compare/v1.3.0...v2.0.0) (2024-01-15)
 
-* added timeout for destination as flag ([f731aac](https://github.com/flowcore-io/flowcore-cli/commit/f731aac025262179af1fd56e2ab2477a674635f9))
 
+### ⚠ BREAKING CHANGES
 
-### Bug Fixes
+* default mode now does not scan all time buckets but fetches populated ones from the index
 
-* added description to start that includes week ([58687a7](https://github.com/flowcore-io/flowcore-cli/commit/58687a7bbb66aaa5d6da26af88e555cbb1e72467))
+### Features
+
+* added fetch of time bucket indexes to speed up fetching events, set --scan to query all time buckets within the requested range. ([650a7cd](https://github.com/flowcore-io/flowcore-cli/commit/650a7cd120ab372148abc79e3c105eee2687b466))
 
 ## [1.2.0](https://github.com/flowcore-io/flowcore-cli/compare/v1.1.2...v1.2.0) (2024-01-09)
 

package/README.md

@@ -17,7 +17,7 @@ $ npm install -g @flowcore/cli
 $ flowcore COMMAND
 running command...
 $ flowcore (--version)
-@flowcore/cli/1.3.0 linux-x64 node-v20.10.0
+@flowcore/cli/2.0.1 linux-x64 node-v20.10.0
 $ flowcore --help [COMMAND]
 USAGE
   $ flowcore COMMAND
@@ -103,7 +103,7 @@ EXAMPLES
   $ flowcore config set -l https://auth.flowcore.io/realms/flowcore/.well-known/openid-configuration -c my-client-id -p
 ```
 
-_See code: [src/commands/config/set.ts](https://github.com/flowcore-io/flowcore-cli/blob/v1.3.0/src/commands/config/set.ts)_
+_See code: [src/commands/config/set.ts](https://github.com/flowcore-io/flowcore-cli/blob/v2.0.1/src/commands/config/set.ts)_
 
 ## `flowcore config show`
 
@@ -123,7 +123,7 @@ EXAMPLES
   $ flowcore config show
 ```
 
-_See code: [src/commands/config/show.ts](https://github.com/flowcore-io/flowcore-cli/blob/v1.3.0/src/commands/config/show.ts)_
+_See code: [src/commands/config/show.ts](https://github.com/flowcore-io/flowcore-cli/blob/v2.0.1/src/commands/config/show.ts)_
 
 ## `flowcore help [COMMANDS]`
 
@@ -166,7 +166,7 @@ EXAMPLES
   $ flowcore login --port 8080
 ```
 
-_See code: [src/commands/login.ts](https://github.com/flowcore-io/flowcore-cli/blob/v1.3.0/src/commands/login.ts)_
+_See code: [src/commands/login.ts](https://github.com/flowcore-io/flowcore-cli/blob/v2.0.1/src/commands/login.ts)_
 
 ## `flowcore plugins`
 
@@ -449,12 +449,14 @@ Stream events from a datacore running on the Flowcore Platform
 
 ```
 USAGE
-  $ flowcore stream STREAM [--profile <value>] [-d <value>] [-j] [-l] [-o http|log] [-s <value>] [-t <value>]
+  $ flowcore stream STREAM [--profile <value>] [-d <value>] [-j] [-l] [-o http|log] [-c] [-s <value>] [-t
+    <value>]
 
 ARGUMENTS
   STREAM  stream url to connect to
 
 FLAGS
+  -c, --scan                 Scan the full time range
   -d, --destination=<value>  [default: http://localhost:3000/transform] Destination to send events to
   -j, --json                 Output json only
   -l, --[no-]live            Change to live mode when reaching last time bucket
@@ -477,7 +479,7 @@ EXAMPLES
   $ flowcore stream https://flowcore.io/<org>/<data core>/<flow type>/[<event type1>,<event type2>,<event type3>].stream -o log -s 3m
 ```
 
-_See code: [src/commands/stream.ts](https://github.com/flowcore-io/flowcore-cli/blob/v1.3.0/src/commands/stream.ts)_
+_See code: [src/commands/stream.ts](https://github.com/flowcore-io/flowcore-cli/blob/v2.0.1/src/commands/stream.ts)_
 
 ## `flowcore version`
 
@@ -514,5 +516,5 @@ DESCRIPTION
   Check what user you are logged in as
 ```
 
-_See code: [src/commands/whoami.ts](https://github.com/flowcore-io/flowcore-cli/blob/v1.3.0/src/commands/whoami.ts)_
+_See code: [src/commands/whoami.ts](https://github.com/flowcore-io/flowcore-cli/blob/v2.0.1/src/commands/whoami.ts)_
 <!-- commandsstop -->

package/dist/commands/stream.d.ts

@@ -10,10 +10,14 @@ export default class Stream extends BaseCommand<typeof Stream> {
         json: import("@oclif/core/lib/interfaces/parser.js").BooleanFlag<boolean>;
         live: import("@oclif/core/lib/interfaces/parser.js").BooleanFlag<boolean>;
         output: import("@oclif/core/lib/interfaces/parser.js").OptionFlag<string, import("@oclif/core/lib/interfaces/parser.js").CustomOptions>;
+        scan: import("@oclif/core/lib/interfaces/parser.js").BooleanFlag<boolean>;
         start: import("@oclif/core/lib/interfaces/parser.js").OptionFlag<string | undefined, import("@oclif/core/lib/interfaces/parser.js").CustomOptions>;
         timeout: import("@oclif/core/lib/interfaces/parser.js").OptionFlag<number, import("@oclif/core/lib/interfaces/parser.js").CustomOptions>;
     };
     run(): Promise<void>;
+    private calculateBuckets;
+    private calculateTimeDifferenceInHours;
+    private fetchTimeBuckets;
     private processEvents;
     private setTimeBucket;
     private streamEvents;

package/dist/commands/stream.js

@@ -3,13 +3,15 @@ import { Args, Flags, ux } from '@oclif/core';
 import dayjs from "dayjs";
 import isSameOrBefore from "dayjs/plugin/isSameOrBefore.js";
 import utc from "dayjs/plugin/utc.js";
+import _ from "lodash";
 import { Subject } from "rxjs";
 import { BaseCommand } from "../base-command.js";
 import { QueryGraphQL } from "../utils/graphql.util.js";
 import { FETCH_DATA_CORE_GQL_QUERY } from "../utils/queries/fetch-data-core.gql.js";
 import { FETCH_EVENT_TYPE_RANGE_GQL_QUERY } from "../utils/queries/fetch-event-type-range.gql.js";
 import { FETCH_EVENTS_GQL_QUERY } from "../utils/queries/fetch-events.gql.js";
-import { TIME_BUCKET_HOUR_PATTERN, createTimebucket, createTimebucketDayjs } from "../utils/timebucket.util.js";
+import { FETCH_CATALOG_INDEXES_QUERY } from "../utils/queries/fetch-indexes.gql.js";
+import { TIME_BUCKET_HOUR_PATTERN, TIME_BUCKET_PATTERN, createTimebucket, createTimebucketDayjs } from "../utils/timebucket.util.js";
 import { LOGIN_CODES, ValidateLogin } from "../utils/validate-login.util.js";
 dayjs.extend(utc);
 dayjs.extend(isSameOrBefore);
@@ -29,6 +31,7 @@ export default class Stream extends BaseCommand {
         json: Flags.boolean({ char: 'j', description: 'Output json only' }),
         live: Flags.boolean({ allowNo: true, char: 'l', default: true, description: 'Change to live mode when reaching last time bucket' }),
         output: Flags.string({ char: 'o', default: 'http', description: 'Output format', options: ['http', 'log'] }),
+        scan: Flags.boolean({ char: 'c', default: false, description: 'Scan the full time range' }),
         start: Flags.string({ char: 's', description: 'Start time bucket to stream from, example: (1y, 1m, 1w, 1d, 1h)' }),
         timeout: Flags.integer({ char: 't', default: 5000, description: 'Timeout in milliseconds to wait for a response from the destination' }),
     };
@@ -62,6 +65,9 @@ export default class Stream extends BaseCommand {
             dataCore,
             organization: org,
         });
+        if (fetchDataCoreResponse.organization.datacores.length === 0) {
+            ux.error(`Data core ${dataCore} not found for organization ${org}`);
+        }
         const dataCoreId = fetchDataCoreResponse.organization.datacores[0].id;
         const eventRangeRequest = await graphqlClient.client.request(FETCH_EVENT_TYPE_RANGE_GQL_QUERY, {
             aggregator,
@@ -91,6 +97,55 @@ export default class Stream extends BaseCommand {
         void this.streamEvents(dataCoreId, aggregator, eventTypes, firstTimeBucket, lastTimeBucket, graphqlClient, observer, flags.live);
         await this.processEvents(observer, flags.destination, flags.output);
     }
+    calculateBuckets(timediff, startTimeBucket) {
+        return _.times(timediff + 1, (n) => dayjs(startTimeBucket).add(n, "hour").format(TIME_BUCKET_PATTERN));
+    }
+    calculateTimeDifferenceInHours(lastTimeBucket, startTimeBucket, firstTimeBucket) {
+        const timediff = dayjs(lastTimeBucket, TIME_BUCKET_HOUR_PATTERN).diff(startTimeBucket, "hour");
+        if (timediff < 0) {
+            ux.error(`Invalid time range, start time bucket (${ux.colorize("yellow", firstTimeBucket)}) is after last time bucket (${ux.colorize("yellow", lastTimeBucket)})`);
+        }
+        return timediff;
+    }
+    async fetchTimeBuckets(dataCoreId, aggregator, eventTypes, startTimeBucket, lastTimeBucket, graphqlClient) {
+        !this.flags.json && ux.action.start("Fetching time buckets");
+        let timeBuckets = [];
+        for (const eventType of eventTypes) {
+            let cursor = null;
+            do {
+                // eslint-disable-next-line no-await-in-loop
+                await graphqlClient.validateToken();
+                try {
+                    // eslint-disable-next-line no-await-in-loop
+                    const response = await graphqlClient.client.request(FETCH_CATALOG_INDEXES_QUERY, {
+                        aggregator,
+                        cursor: cursor || undefined,
+                        dataCoreId,
+                        eventType,
+                        fromTimeBucket: startTimeBucket.format(TIME_BUCKET_PATTERN),
+                        toTimeBucket: lastTimeBucket,
+                    });
+                    cursor = response.datacore.fetchIndexes.cursor;
+                    timeBuckets.push(...response.datacore.fetchIndexes.timeBuckets);
+                }
+                catch (error) {
+                    // don't display error if token is expired
+                    if (error instanceof Error && error.message.includes("Token is expired")) {
+                        continue;
+                    }
+                    this.warn(`Error fetching events: ${error}`);
+                }
+            } while (cursor);
+        }
+        timeBuckets = _.uniq(timeBuckets);
+        !this.flags.json && ux.action.stop(`Found ${timeBuckets.length} time buckets`);
+        if (timeBuckets.length === 0) {
+            !this.flags.json && this.warn("No time buckets found, setting to scan based on requested time range");
+            const timediff = this.calculateTimeDifferenceInHours(lastTimeBucket, startTimeBucket, createTimebucket(dayjs()));
+            timeBuckets = this.calculateBuckets(timediff, startTimeBucket);
+        }
+        return timeBuckets;
+    }
     async processEvents(observer, destination, output) {
         let done = false;
         let events = [];
@@ -181,13 +236,19 @@ export default class Stream extends BaseCommand {
         }
     }
     async streamEvents(dataCoreId, aggregator, eventTypes, firstTimeBucket, lastTimeBucket, graphqlClient, observer, live) {
-        let startTimeBucket = dayjs(firstTimeBucket, TIME_BUCKET_HOUR_PATTERN);
+        const startTimeBucket = dayjs(firstTimeBucket, TIME_BUCKET_HOUR_PATTERN);
         const currentTime = dayjs.utc();
         let liveMode = false;
         let lastEventId = null;
+        const timediff = this.calculateTimeDifferenceInHours(lastTimeBucket, startTimeBucket, firstTimeBucket);
+        let timeBuckets = [];
+        timeBuckets = this.flags.scan ? this.calculateBuckets(timediff, startTimeBucket) : (await this.fetchTimeBuckets(dataCoreId, aggregator, eventTypes, startTimeBucket, lastTimeBucket, graphqlClient));
         // eslint-disable-next-line no-constant-condition
         while (true) {
-            const timeBucket = createTimebucket(startTimeBucket);
+            const timeBucket = timeBuckets.shift();
+            if (!timeBucket) {
+                ux.error("No time bucket found");
+            }
             if (!this.flags.json) {
                 ux.action.start(`Fetching events for ${startTimeBucket}`);
             }
@@ -204,6 +265,7 @@ export default class Stream extends BaseCommand {
                     const response = await graphqlClient.client.request(FETCH_EVENTS_GQL_QUERY, {
                         afterEventId: lastEventId || undefined,
                         aggregator,
+                        cursor: cursor || undefined,
                         dataCoreId,
                         eventTypes,
                         timeBucket,
@@ -223,10 +285,7 @@ export default class Stream extends BaseCommand {
                     this.warn(`Error fetching events: ${error}`);
                 }
             } while (cursor);
-            if (dayjs(timeBucket, TIME_BUCKET_HOUR_PATTERN).isBefore(currentTime, "hour")) {
-                startTimeBucket = startTimeBucket.add(1, "hour");
-            }
-            if (timeBucket === lastTimeBucket) {
+            if (timeBuckets.length === 0) {
                 if (!live) {
                     observer.complete();
                     if (!this.flags.json) {
@@ -236,9 +295,9 @@ export default class Stream extends BaseCommand {
                 }
                 if (!liveMode) {
                     !this.flags.json && this.warn("Reached last time bucket, switching to live mode");
-                    startTimeBucket = createTimebucketDayjs(currentTime);
                     liveMode = true;
                 }
+                timeBuckets.push(createTimebucketDayjs(currentTime).format(TIME_BUCKET_PATTERN));
             }
             if (liveMode && !found) {
                 // eslint-disable-next-line no-await-in-loop

package/dist/utils/queries/fetch-indexes.gql.d.ts

@@ -0,0 +1,18 @@
+export type FetchCatalogIndexesResponse = {
+    datacore: {
+        fetchIndexes: {
+            cursor: string;
+            timeBuckets: string[];
+        };
+    };
+};
+export type FetchCatalogIndexesInput = {
+    aggregator: string;
+    cursor?: string;
+    dataCoreId: string;
+    eventType: string;
+    fromTimeBucket?: string;
+    pageSize?: number;
+    toTimeBucket?: string;
+};
+export declare const FETCH_CATALOG_INDEXES_QUERY: string;

package/dist/utils/queries/fetch-indexes.gql.js

@@ -0,0 +1,18 @@
+import { gql } from "graphql-request";
+export const FETCH_CATALOG_INDEXES_QUERY = gql `
+  query FLOWCORE_CLI_FETCH_CATALOG_INDEXES($dataCoreId: ID!, $aggregator: String!, $eventType: String!, $cursor: String, $fromTimeBucket: String, $toTimeBucket: String, $pageSize: Int) {
+    datacore(search: {id: $dataCoreId}) {
+      fetchIndexes(input: {
+        aggregator: $aggregator,
+        eventType: $eventType,
+        cursor: $cursor
+        fromTimeBucket: $fromTimeBucket
+        toTimeBucket: $toTimeBucket
+        pageSize: $pageSize
+      }) {
+        timeBuckets
+        cursor
+      }
+    }
+  }
+`;

package/oclif.manifest.json

@@ -101,6 +101,13 @@
           ],
           "type": "option"
         },
+        "scan": {
+          "char": "c",
+          "description": "Scan the full time range",
+          "name": "scan",
+          "allowNo": false,
+          "type": "boolean"
+        },
         "start": {
           "char": "s",
           "description": "Start time bucket to stream from, example: (1y, 1m, 1w, 1d, 1h)",
@@ -268,5 +275,5 @@
       ]
     }
   },
-  "version": "1.3.0"
+  "version": "2.0.1"
 }

package/package.json

@@ -86,7 +86,7 @@
     "prestart": "npm run build",
     "update-schema": "rover graph introspect https://graph.api.staging.flowcore.io/graphql -o schema.gql"
   },
-  "version": "1.3.0",
+  "version": "2.0.1",
   "bugs": "https://github.com/flowcore-io/flowcore-cli/issues",
   "keywords": [
     "flowcore",