@flowcore/cli 2.6.0 → 2.8.0

package/CHANGELOG.md

@@ -10,6 +10,20 @@
 
 * added description to start that includes week ([58687a7](https://github.com/flowcore-io/flowcore-cli/commit/58687a7bbb66aaa5d6da26af88e555cbb1e72467))
 
+## [2.8.0](https://github.com/flowcore-io/flowcore-cli/compare/v2.7.0...v2.8.0) (2024-02-24)
+
+
+### Features
+
+* add now option to start arg to start streaming from now ([8070602](https://github.com/flowcore-io/flowcore-cli/commit/80706025d0a3ce0c9a0e57294d39eb530b34d3f9))
+
+## [2.7.0](https://github.com/flowcore-io/flowcore-cli/compare/v2.6.0...v2.7.0) (2024-02-22)
+
+
+### Features
+
+* add generate manifest commands for scenario and data core ([551d80f](https://github.com/flowcore-io/flowcore-cli/commit/551d80f47e6537cb26e56ce9f69f4b25bb554ef2))
+
 ## [2.6.0](https://github.com/flowcore-io/flowcore-cli/compare/v2.5.2...v2.6.0) (2024-02-22)
 
 

package/README.md

@@ -18,7 +18,7 @@ $ npm install -g @flowcore/cli
 $ flowcore COMMAND
 running command...
 $ flowcore (--version)
-@flowcore/cli/2.6.0 linux-x64 node-v20.11.1
+@flowcore/cli/2.8.0 linux-x64 node-v20.11.1
 $ flowcore --help [COMMAND]
 USAGE
   $ flowcore COMMAND
@@ -32,6 +32,9 @@ USAGE
 * [`flowcore config set`](#flowcore-config-set)
 * [`flowcore config show`](#flowcore-config-show)
 * [`flowcore data-core apply`](#flowcore-data-core-apply)
+* [`flowcore data-core generate event-type FLOWTYPE`](#flowcore-data-core-generate-event-type-flowtype)
+* [`flowcore data-core generate flow-type`](#flowcore-data-core-generate-flow-type)
+* [`flowcore data-core generate manifest`](#flowcore-data-core-generate-manifest)
 * [`flowcore help [COMMANDS]`](#flowcore-help-commands)
 * [`flowcore login`](#flowcore-login)
 * [`flowcore plugins`](#flowcore-plugins)
@@ -45,6 +48,8 @@ USAGE
 * [`flowcore plugins:uninstall PLUGIN...`](#flowcore-pluginsuninstall-plugin-2)
 * [`flowcore plugins update`](#flowcore-plugins-update)
 * [`flowcore scenario apply`](#flowcore-scenario-apply)
+* [`flowcore scenario generate manifest`](#flowcore-scenario-generate-manifest)
+* [`flowcore scenario generate transformer`](#flowcore-scenario-generate-transformer)
 * [`flowcore scenario local`](#flowcore-scenario-local)
 * [`flowcore stream STREAM`](#flowcore-stream-stream)
 * [`flowcore version`](#flowcore-version)
@@ -70,7 +75,7 @@ EXAMPLES
   $ flowcore apply -f ./path/to/manifest.yml
 ```
 
-_See code: [src/commands/apply.ts](https://github.com/flowcore-io/flowcore-cli/blob/v2.6.0/src/commands/apply.ts)_
+_See code: [src/commands/apply.ts](https://github.com/flowcore-io/flowcore-cli/blob/v2.8.0/src/commands/apply.ts)_
 
 ## `flowcore autocomplete [SHELL]`
 
@@ -180,7 +185,93 @@ EXAMPLES
   $ cat <<EOF | flowcore data-core apply -f -
 ```
 
-_See code: [@flowcore/cli-plugin-data-core](https://github.com/flowcore/flowcore-cli-data-core/blob/v2.0.0/src/commands/data-core/apply.ts)_
+_See code: [@flowcore/cli-plugin-data-core](https://github.com/flowcore/flowcore-cli-data-core/blob/v2.1.0/src/commands/data-core/apply.ts)_
+
+## `flowcore data-core generate event-type FLOWTYPE`
+
+add a event type to a data core manifest
+
+```
+USAGE
+  $ flowcore data-core generate event-type FLOWTYPE [--profile <value>] [-d <value>] [-f <value>] [-n <value>]
+
+FLAGS
+  -d, --description=<value>  description of the event type
+  -f, --file=<value>         file to modify
+  -n, --name=<value>         name of the event type to generate
+      --profile=<value>      Specify the configuration profile to use
+
+DESCRIPTION
+  add a event type to a data core manifest
+
+EXAMPLES
+  $ flowcore data-core generate event-type flow-type-name -n event-type-name
+
+  $ flowcore data-core generate event-type flow-type-name -n event-type-name -d "description of the event type"
+
+  $ flowcore data-core generate event-type flow-type-name -n event-type-name -d "description of the event type" -f example.yaml
+```
+
+_See code: [@flowcore/cli-plugin-data-core](https://github.com/flowcore/flowcore-cli-data-core/blob/v2.1.0/src/commands/data-core/generate/event-type.ts)_
+
+## `flowcore data-core generate flow-type`
+
+add a flow type to a data core manifest
+
+```
+USAGE
+  $ flowcore data-core generate flow-type [--profile <value>] [-d <value>] [-f <value>] [-n <value>]
+
+FLAGS
+  -d, --description=<value>  description of the flow type
+  -f, --file=<value>         file to modify
+  -n, --name=<value>         name of the flow type to generate
+      --profile=<value>      Specify the configuration profile to use
+
+DESCRIPTION
+  add a flow type to a data core manifest
+
+EXAMPLES
+  $ flowcore data-core generate flow-type -n flow-type-name
+
+  $ flowcore data-core generate flow-type -n flow-type-name -d "description of the flow type"
+
+  $ flowcore data-core generate flow-type -n flow-type-name -d "description of the flow type" -f example.yaml
+```
+
+_See code: [@flowcore/cli-plugin-data-core](https://github.com/flowcore/flowcore-cli-data-core/blob/v2.1.0/src/commands/data-core/generate/flow-type.ts)_
+
+## `flowcore data-core generate manifest`
+
+Generate a data core manifest
+
+```
+USAGE
+  $ flowcore data-core generate manifest -t <value> [--profile <value>] [-f <value>] [-n <value>] [-o] [--placeholder]
+
+FLAGS
+  -f, --file=<value>     file to apply
+  -n, --name=<value>     name of the data core to generate
+  -o, --overwrite        overwrite the existing data core
+  -t, --tenant=<value>   (required) tenant to apply the data core to, this is the org for your organization, it can be
+                         seen in the url when accessing your organization
+      --placeholder      generate a placeholder manifest
+      --profile=<value>  Specify the configuration profile to use
+
+DESCRIPTION
+  Generate a data core manifest
+
+EXAMPLES
+  $ flowcore data-core generate manifest -t flowcore
+
+  $ flowcore data-core generate manifest -t flowcore --placeholder
+
+  $ flowcore data-core generate manifest -t flowcore -f example.yaml
+
+  $ flowcore data-core generate manifest -t flowcore -n data-core-name -f example.yaml
+```
+
+_See code: [@flowcore/cli-plugin-data-core](https://github.com/flowcore/flowcore-cli-data-core/blob/v2.1.0/src/commands/data-core/generate/manifest.ts)_
 
 ## `flowcore help [COMMANDS]`
 
@@ -532,7 +623,66 @@ EXAMPLES
   $ cat <<EOF | flowcore scenario apply -f -
 ```
 
-_See code: [@flowcore/cli-plugin-scenario](https://github.com/flowcore/flowcore-cli-plugin-scenario/blob/v2.3.0/src/commands/scenario/apply.ts)_
+_See code: [@flowcore/cli-plugin-scenario](https://github.com/flowcore/flowcore-cli-plugin-scenario/blob/v2.5.0/src/commands/scenario/apply.ts)_
+
+## `flowcore scenario generate manifest`
+
+Generate a scenario manifest
+
+```
+USAGE
+  $ flowcore scenario generate manifest -t <value> [--profile <value>] [-f <value>] [-n <value>] [-o] [--placeholder]
+
+FLAGS
+  -f, --file=<value>     file to apply
+  -n, --name=<value>     name of the scenario to generate
+  -o, --overwrite        overwrite the existing scenario
+  -t, --tenant=<value>   (required) tenant to apply the scenario to, this is the org for your organization, it can be
+                         seen in the url when accessing your organization
+      --placeholder      generate a placeholder manifest
+      --profile=<value>  Specify the configuration profile to use
+
+DESCRIPTION
+  Generate a scenario manifest
+
+EXAMPLES
+  $ flowcore scenario generate manifest -t flowcore
+
+  $ flowcore scenario generate manifest -t flowcore --placeholder
+
+  $ flowcore scenario generate manifest -t flowcore -f example.yaml
+
+  $ flowcore scenario generate manifest -t flowcore -n scenario-name -f example.yaml
+```
+
+_See code: [@flowcore/cli-plugin-scenario](https://github.com/flowcore/flowcore-cli-plugin-scenario/blob/v2.5.0/src/commands/scenario/generate/manifest.ts)_
+
+## `flowcore scenario generate transformer`
+
+add a transformer to a scenario manifest
+
+```
+USAGE
+  $ flowcore scenario generate transformer [--profile <value>] [-d <value>] [-f <value>] [-n <value>]
+
+FLAGS
+  -d, --description=<value>  description of the transformer
+  -f, --file=<value>         file to modify
+  -n, --name=<value>         name of the transformer to generate
+      --profile=<value>      Specify the configuration profile to use
+
+DESCRIPTION
+  add a transformer to a scenario manifest
+
+EXAMPLES
+  $ flowcore scenario generate transformer -n flow-type-name
+
+  $ flowcore scenario generate transformer -n flow-type-name -d "description of the transformer"
+
+  $ flowcore scenario generate transformer -n flow-type-name -d "description of the transformer" -f example.yaml
+```
+
+_See code: [@flowcore/cli-plugin-scenario](https://github.com/flowcore/flowcore-cli-plugin-scenario/blob/v2.5.0/src/commands/scenario/generate/transformer.ts)_
 
 ## `flowcore scenario local`
 
@@ -540,18 +690,20 @@ Spin up local stream threads based on a scenario manifest
 
 ```
 USAGE
-  $ flowcore scenario local -e <value> -f <value> [--profile <value>] [-m http] [-c] [-s <value>] [-t <value>] [-y]
+  $ flowcore scenario local -e <value> -f <value> [--profile <value>] [-H <value>] [-m http] [-c] [-s <value>] [-t
+    <value>] [-y]
 
 FLAGS
-  -c, --scan              Scan the full time range
-  -e, --endpoint=<value>  (required) stream endpoint
-  -f, --file=<value>...   (required) file to apply
-  -m, --mode=<option>     [default: http] stream mode
-                          <options: http>
-  -s, --start=<value>     Start time bucket to stream from, example: (1y, 1m, 1w, 1d, 1h)
-  -t, --timeout=<value>   [default: 5000] Timeout in milliseconds to wait for a response from the destination
-  -y, --yes               skip confirmation
-      --profile=<value>   Specify the configuration profile to use
+  -H, --header=<value>...  [default: ] header to send with the request, example: (-H 'Authorization: Bearer TOKEN')
+  -c, --scan               Scan the full time range
+  -e, --endpoint=<value>   (required) stream endpoint
+  -f, --file=<value>...    (required) file to apply
+  -m, --mode=<option>      [default: http] stream mode
+                           <options: http>
+  -s, --start=<value>      Start time bucket to stream from, example: (1y, 1m, 1w, 1d, 1h)
+  -t, --timeout=<value>    [default: 5000] Timeout in milliseconds to wait for a response from the destination
+  -y, --yes                skip confirmation
+      --profile=<value>    Specify the configuration profile to use
 
 DESCRIPTION
   Spin up local stream threads based on a scenario manifest
@@ -562,7 +714,7 @@ EXAMPLES
   $ cat <<EOF | flowcore scenario local -f -
 ```
 
-_See code: [@flowcore/cli-plugin-scenario](https://github.com/flowcore/flowcore-cli-plugin-scenario/blob/v2.3.0/src/commands/scenario/local.ts)_
+_See code: [@flowcore/cli-plugin-scenario](https://github.com/flowcore/flowcore-cli-plugin-scenario/blob/v2.5.0/src/commands/scenario/local.ts)_
 
 ## `flowcore stream STREAM`
 
@@ -584,7 +736,7 @@ FLAGS
   -l, --[no-]live            Change to live mode when reaching last time bucket
   -o, --output=<option>      [default: http] Output format
                              <options: http|log>
-  -s, --start=<value>        Start time bucket to stream from, example: (1y, 1m, 1w, 1d, 1h)
+  -s, --start=<value>        Start time bucket to stream from, example: (1y, 1m, 1w, 1d, 1h, now)
   -t, --timeout=<value>      [default: 5000] Timeout in milliseconds to wait for a response from the destination
       --profile=<value>      Specify the configuration profile to use
 
@@ -601,7 +753,7 @@ EXAMPLES
   $ flowcore stream https://flowcore.io/<org>/<data core>/<flow type>/[<event type1>,<event type2>,<event type3>].stream -o log -s 3m
 ```
 
-_See code: [src/commands/stream.ts](https://github.com/flowcore-io/flowcore-cli/blob/v2.6.0/src/commands/stream.ts)_
+_See code: [src/commands/stream.ts](https://github.com/flowcore-io/flowcore-cli/blob/v2.8.0/src/commands/stream.ts)_
 
 ## `flowcore version`
 

package/dist/commands/stream.js

@@ -1,6 +1,7 @@
 import { Queue } from "@datastructures-js/queue";
 import { BaseCommand, LOGIN_CODES, ValidateLogin } from "@flowcore/cli-plugin-config";
 import { Args, Flags, ux } from '@oclif/core';
+import { TimeUuid } from "cassandra-uuid";
 import dayjs from "dayjs";
 import isSameOrBefore from "dayjs/plugin/isSameOrBefore.js";
 import utc from "dayjs/plugin/utc.js";
@@ -32,7 +33,7 @@ export default class Stream extends BaseCommand {
         live: Flags.boolean({ allowNo: true, char: 'l', default: true, description: 'Change to live mode when reaching last time bucket' }),
         output: Flags.string({ char: 'o', default: 'http', description: 'Output format', options: ['http', 'log'] }),
         scan: Flags.boolean({ char: 'c', default: false, description: 'Scan the full time range' }),
-        start: Flags.string({ char: 's', description: 'Start time bucket to stream from, example: (1y, 1m, 1w, 1d, 1h)' }),
+        start: Flags.string({ char: 's', description: 'Start time bucket to stream from, example: (1y, 1m, 1w, 1d, 1h, now)' }),
         timeout: Flags.integer({ char: 't', default: 5000, description: 'Timeout in milliseconds to wait for a response from the destination' }),
     };
     async run() {
@@ -84,11 +85,17 @@ export default class Stream extends BaseCommand {
             dataCoreId,
         });
         let firstTimeBucket = eventRangeRequest.datacore.flowtypes.find(ft => ft.aggregator === aggregator)?.events.find(e => eventTypes.includes(e.name))?.catalog.range.firstTimeBucket;
+        let afterEventId;
         if (!firstTimeBucket && !flags.start) {
             firstTimeBucket = createTimebucket(dayjs());
             !this.flags.json && this.warn(`First time bucket not found, setting to current time (${ux.colorize("yellowBright", firstTimeBucket)})`);
         }
-        if (flags.start) {
+        if (flags.start === "now") {
+            afterEventId = TimeUuid.now();
+            firstTimeBucket = createTimebucket(dayjs());
+            !this.flags.json && this.log(ux.colorize("green", `Setting first time bucket and event id to now (${ux.colorize("yellowBright", firstTimeBucket)}), (${ux.colorize("yellowBright", afterEventId)})`));
+        }
+        else if (flags.start) {
             firstTimeBucket = this.setTimeBucket(dayjs(), flags.start);
             !this.flags.json && this.log(ux.colorize("green", `Setting first time bucket to ${ux.colorize("yellow", flags.start)} ago (${ux.colorize("yellowBright", firstTimeBucket)})`));
         }
@@ -96,7 +103,9 @@ export default class Stream extends BaseCommand {
             firstTimeBucket = this.setTimeBucket(dayjs(), "1d");
             !this.flags.json && this.log(ux.colorize("green", `Setting first time bucket to ${ux.colorize("yellow", "1d")} ago (${ux.colorize("yellowBright", firstTimeBucket)})`));
         }
-        let lastTimeBucket = eventRangeRequest.datacore.flowtypes.find(ft => ft.aggregator === aggregator)?.events.find(e => eventTypes.includes(e.name))?.catalog.range.lastTimeBucket;
+        let lastTimeBucket = flags.start === "now"
+            ? firstTimeBucket
+            : eventRangeRequest.datacore.flowtypes.find(ft => ft.aggregator === aggregator)?.events.find(e => eventTypes.includes(e.name))?.catalog.range.lastTimeBucket;
         if (!lastTimeBucket) {
             lastTimeBucket = createTimebucket(dayjs());
             !this.flags.json && this.warn(`Last time bucket not found, setting to current time (${ux.colorize("yellowBright", lastTimeBucket)})`);
@@ -104,7 +113,7 @@ export default class Stream extends BaseCommand {
         const observer = new Subject();
         !this.flags.json && this.log(ux.colorize("blackBright", `Starting to stream events for ${dataCoreId} - ${aggregator} - ${eventTypes}, press ${ux.colorize("whiteBright", "ctrl+c")} to stop`));
         // eslint-disable-next-line no-void
-        void this.streamEvents(dataCoreId, aggregator, eventTypes, firstTimeBucket, lastTimeBucket, graphqlClient, observer, flags.live);
+        void this.streamEvents(dataCoreId, aggregator, eventTypes, firstTimeBucket, lastTimeBucket, graphqlClient, observer, flags.live, afterEventId);
         await this.processEvents(observer, flags.destination, flags.output, flags.header);
     }
     calculateBuckets(timediff, startTimeBucket) {
@@ -251,11 +260,11 @@ export default class Stream extends BaseCommand {
             }
         }
     }
-    async streamEvents(dataCoreId, aggregator, eventTypes, firstTimeBucket, lastTimeBucket, graphqlClient, observer, live) {
+    async streamEvents(dataCoreId, aggregator, eventTypes, firstTimeBucket, lastTimeBucket, graphqlClient, observer, live, afterEventId) {
         const startTimeBucket = dayjs(firstTimeBucket, TIME_BUCKET_HOUR_PATTERN);
         const currentTime = dayjs.utc();
         let liveMode = false;
-        let lastEventId = null;
+        let lastEventId = afterEventId ?? null;
         const timediff = this.calculateTimeDifferenceInHours(lastTimeBucket, startTimeBucket, firstTimeBucket);
         let timeBuckets = [];
         timeBuckets = this.flags.scan ? this.calculateBuckets(timediff, startTimeBucket) : (await this.fetchTimeBuckets(dataCoreId, aggregator, eventTypes, startTimeBucket, lastTimeBucket, graphqlClient));

package/oclif.manifest.json

@@ -125,7 +125,7 @@
         },
         "start": {
           "char": "s",
-          "description": "Start time bucket to stream from, example: (1y, 1m, 1w, 1d, 1h)",
+          "description": "Start time bucket to stream from, example: (1y, 1m, 1w, 1d, 1h, now)",
           "name": "start",
           "hasDynamicHelp": false,
           "multiple": false,
@@ -157,5 +157,5 @@
       ]
     }
   },
-  "version": "2.6.0"
+  "version": "2.8.0"
 }

package/package.json

@@ -6,8 +6,8 @@
   "dependencies": {
     "@datastructures-js/queue": "^4.2.3",
     "@flowcore/cli-plugin-config": "^1.0.5",
-    "@flowcore/cli-plugin-data-core": "^2.0.0",
-    "@flowcore/cli-plugin-scenario": "^2.3.0",
+    "@flowcore/cli-plugin-data-core": "^2.1.0",
+    "@flowcore/cli-plugin-scenario": "^2.5.0",
     "@flowcore/time-bucket": "^1.1.0",
     "@oclif/core": "^3",
     "@oclif/plugin-autocomplete": "^3.0.2",
@@ -16,6 +16,7 @@
     "@oclif/plugin-plugins": "^4",
     "@oclif/plugin-version": "^2.0.8",
     "axios": "^1.6.2",
+    "cassandra-uuid": "^0.1.0",
     "cross-fetch": "^4.0.0",
     "dayjs": "^1.11.10",
     "enquirer": "^2.4.1",
@@ -89,7 +90,7 @@
     "prestart": "npm run build",
     "update-schema": "rover graph introspect https://graph.api.staging.flowcore.io/graphql -o schema.gql"
   },
-  "version": "2.6.0",
+  "version": "2.8.0",
   "bugs": "https://github.com/flowcore-io/flowcore-cli/issues",
   "keywords": [
     "flowcore",