@streamr/cli-tools 6.0.0-alpha.0 → 6.0.0-alpha.4

package/.eslintignore

@@ -1,3 +1,2 @@
 node_modules/**
 dist/**
-

package/CHANGELOG.md

@@ -7,9 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 - Add permission commands: `stream grant-permission` and `stream revoke-permission`
 - Remove `typescript` and `ts-node` as run-time dependencies
-- Remove `--msg-chain-id` parameter from `stream resend from`
-- (Breaking) `streamr stream create` argument is a stream ID, not a name
+- (Breaking) Remove `--msg-chain-id` parameter from `stream resend from`
+- (Breaking) `stream create` argument is a stream ID, not a name
 - Support path notation when defining a stream ID
+- (Breaking) Remove `misc get-session-token` command
+- (Breaking) Change permission names and arguments:
+  - renamed `--operation` to `--permission` in `stream list`
+  - the data format of `permissions` returned by `--include-permissions` of `stream show` 
+  - enum values updated for all commands: `subscribe`, `publish`, `edit`, `delete` and `grant`
+- (Breaking) Remove `lastUpdated` and `name` columns from `stream list` output
+- (Breaking) Modify configuration management:
+  - can read config from `~/.streamr/config/default.json`
+  - `--config` argument added
+  - `--stg`, `--http-url` and `--ws-url` arguments removed
+- (Breaking) Rename `--config` argument to `--stream-config` in `stream create`
+- (Breaking) Remove `stream list` command
+- Add `stream search` command
+- (Breaking) Rename `--partition-key` argument to `--partition-key-field` in `stream publish`
+- Bump dependency streamr-client to 6.1.0
+- Bump dependency commander to 8.3.0
 
 ## [5.0.0] - 2021-05-05
 ### Added

package/README.md

@@ -50,9 +50,6 @@ To subscribe to a private stream and authenticate with an Ethereum private key:
 streamr stream subscribe streamId --private-key <key>
 ```
 
-Flag `--dev` or `--stg` can be enabled for the command to operate on pre-defined development or staging environment. Alternatively, you can give `--ws-url <url>` and `--http-url <url>` to connect to any custom network.
-
-
 ###  publish
 Used to publish events to a stream from stdin line-by-line. Each line should be a valid JSON object.
 
@@ -61,8 +58,6 @@ Example of use:
 streamr stream publish <streamId> --private-key <key>
 ```
 
-Flag `--dev` or `--stg` can be enabled for the command to operate on pre-defined development or staging environment.
-
 
 ### generate
 Generate random JSON objects to stdout line-by-line.
@@ -116,8 +111,28 @@ To fetch data between two date-times
 streamr stream resend range 2019-05-10T17:00:00 2019-05-11T21:00:00 <streamId> --private-key <key>
 ```
 
-Flag `--dev` or `--stg` can be enabled for the command to operate on pre-defined development or staging environment.
+#### Configuration
 
+User can specify environment and authentication details with the following command line arguments:
+- `--private-key <key>`, e.g. `--private-key 0x1234567890123456789012345678901234567890123456789012345678901234`
+- `--config <file>`, e.g. `--config foobar.json`
+- `--dev` use the pre-defined [development environment](https://github.com/streamr-dev/streamr-docker-dev)
+
+The `--config` argument tries to read a configuration file from the current working directory (either without a file extension, or with `.json` extension added). It also tries to read it from `~/.streamr/config/${id}.json` dotfile.
+
+If no `--config` argument is specified, default settings are read from `~/.streamr/config/default.json`, if that file exists.
+
+The configuration file is a JSON. It has one root-level property `client`, which contains any configuration properties for the [streamr-client-javascript](https://github.com/streamr-dev/network-monorepo/blob/main/packages/client/) client. Example:
+```
+{
+    "client": {
+        "auth": {
+            "privateKey": ...
+        },
+        "publishWithSignature": "always"
+    }
+}
+```
 
 ### Examples: Piping with subscribe and publish
 
@@ -157,13 +172,8 @@ the real-time events.
 streamr stream subscribe streamr.eth/demos/helsinki-trams | streamr stream publish --dev <streamId> --private-key <key>
 ```
 
-And the same for staging environment:
-```
-streamr stream subscribe streamr.eth/demos/helsinki-trams | streamr stream publish --stg <streamId> --private-key <key>
-```
-
 ## Develop
-This project is a thin wrapper around [streamr-client-javascript](https://github.com/streamr-dev/streamr-client-javascript),
+This project is a thin wrapper around [streamr-client-javascript](https://github.com/streamr-dev/network-monorepo/blob/main/packages/client/),
 which does the heavy lifting, while this project concentrates on CLI concerns: parsing and
 passing arguments, stdin/stdout, errors, and so forth.
 

package/bin/streamr-mock-data-generate.ts

@@ -1,17 +1,47 @@
 #!/usr/bin/env node
-import { Command } from 'commander'
-import { generate } from '../src/generate'
-import { exitWithHelpIfArgsNotBetween, createFnParseInt } from './common'
 import pkg from '../package.json'
+import { createFnParseInt } from '../src/common'
+import { createCommand } from '../src/command'
 
-const program = new Command()
-program
+// From: https://stackoverflow.com/questions/10726909/random-alpha-numeric-string-in-javascript
+function randomString(
+    length: number,
+    chars = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+): string {
+    let result = ''
+    for (let i = length; i > 0; --i) {
+        result += chars[Math.floor(Math.random() * chars.length)]
+    }
+    return result
+}
+
+function genArray<T>(size: number, elementFn: () => T): T[] {
+    const arr = []
+    for (let i=0; i < size; ++i) {
+        arr.push(elementFn())
+    }
+    return arr
+}
+
+export const generate = (rate: number): void => {
+    setInterval(() => {
+        console.info(JSON.stringify({
+            someText: randomString(64),
+            aNumber: Math.random() * 10000,
+            bNumber: Math.random(),
+            yesOrNo: Math.random() > 0.5,
+            arrayOfStrings: genArray(Math.floor(Math.random() * 20), () => randomString(8)),
+            arrayOfIntegers: genArray(Math.floor(Math.random() * 10), () => Math.floor(Math.random() * 100))
+
+        }))
+    }, rate)
+}
+
+createCommand()
     .description('generate and print semi-random JSON data to stdout')
     .option('-r, --rate <n>', 'rate in milliseconds', createFnParseInt('--rate'), 500)
     .version(pkg.version)
     .action((options: any) => {
         generate(options.rate)
     })
-    .parse(process.argv)
-
-exitWithHelpIfArgsNotBetween(program, 0, 0)
+    .parse()

package/bin/streamr-mock-data.ts

@@ -7,4 +7,4 @@ program
     .usage('<command> [<args>]')
     .description('mock-data subcommands')
     .command('generate', 'generate JSON data')
-    .parse(process.argv)
+    .parse()

package/bin/streamr-storage-node-add-stream.ts

@@ -1,32 +1,12 @@
 #!/usr/bin/env node
-import { Command } from 'commander'
-import { StreamrClient, Stream } from 'streamr-client'
-import {
-    envOptions,
-    authOptions,
-    exitWithHelpIfArgsNotBetween,
-    formStreamrOptionsWithEnv,
-    getStreamId,
-} from './common'
-import pkg from '../package.json'
+import '../src/logLevel'
+import StreamrClient from 'streamr-client'
+import { createClientCommand } from '../src/command'
 
-const program = new Command()
-program
+createClientCommand(async (client: StreamrClient, storageNodeAddress: string, streamId: string) => {
+    const stream = await client.getStream(streamId)
+    await stream.addToStorageNode(storageNodeAddress)
+})
     .arguments('<storageNodeAddress> <streamId>')
     .description('add stream to a storage node')
-authOptions(program)
-envOptions(program)
-    .version(pkg.version)
-    .action((storageNodeAddress: string, streamIdOrPath: string, options: any) => {
-        const client = new StreamrClient(formStreamrOptionsWithEnv(options))
-        const streamId = getStreamId(streamIdOrPath, options)!
-        client.getStream(streamId)
-            .then((stream: Stream) => stream.addToStorageNode(storageNodeAddress))
-            .catch((err) => {
-                console.error(err)
-                process.exit(1)
-            })
-    })
-    .parse(process.argv)
-
-exitWithHelpIfArgsNotBetween(program, 2, 2)
+    .parseAsync()

package/bin/streamr-storage-node-list-stream-parts.ts

@@ -1,38 +1,22 @@
 #!/usr/bin/env node
-import { Command } from 'commander'
-import { StreamrClient } from 'streamr-client'
-import {
-    envOptions,
-    authOptions,
-    exitWithHelpIfArgsNotBetween,
-    formStreamrOptionsWithEnv,
-} from './common'
-import pkg from '../package.json'
+import '../src/logLevel'
 import EasyTable from 'easy-table'
+import StreamrClient from 'streamr-client'
+import { createClientCommand } from '../src/command'
+import { StreamPartIDUtils } from 'streamr-client-protocol'
 
-const program = new Command()
-program
+createClientCommand((async (client: StreamrClient, storageNodeAddress: string) => {
+    const streamParts = await client.getStreamPartsByStorageNode(storageNodeAddress)
+    if (streamParts.length > 0) {
+        console.info(EasyTable.print(streamParts.map((streamPartId) => {
+            const [streamId, streamPartition] = StreamPartIDUtils.getStreamIDAndPartition(streamPartId)
+            return {
+                streamId,
+                streamPartition
+            }
+        })))
+    }
+}))
     .arguments('<storageNodeAddress>')
-    .description('list streams parts in a storage node')
-authOptions(program)
-envOptions(program)
-    .version(pkg.version)
-    .action((storageNodeAddress: string, options: any) => {
-        const client = new StreamrClient(formStreamrOptionsWithEnv(options))
-        client.getStreamPartsByStorageNode(storageNodeAddress)
-            .then((streamParts) => {
-                if (streamParts.length > 0) {
-                    console.info(EasyTable.print(streamParts.map(({ streamId, streamPartition }) => ({
-                        streamId,
-                        streamPartition,
-                    }))))
-                }
-                return true
-            }).catch((err) => {
-                console.error(err)
-                process.exit(1)
-            })
-    })
-    .parse(process.argv)
-
-exitWithHelpIfArgsNotBetween(program, 1, 1)
+    .description('list stream parts in a storage node')
+    .parseAsync()

package/bin/streamr-storage-node-list.ts

@@ -1,52 +1,27 @@
 #!/usr/bin/env node
-import { Command } from 'commander'
+import '../src/logLevel'
 import { StreamrClient } from 'streamr-client'
-import {
-    envOptions,
-    authOptions,
-    exitWithHelpIfArgsNotBetween,
-    formStreamrOptionsWithEnv,
-    getStreamId,
-} from './common'
-import pkg from '../package.json'
 import EasyTable from 'easy-table'
+import { createClientCommand } from '../src/command'
 
 const getStorageNodes = async (streamId: string | undefined, client: StreamrClient): Promise<string[]> => {
     if (streamId !== undefined) {
         const stream = await client.getStream(streamId)
-        const storageNodes = await stream.getStorageNodes()
-        return storageNodes.map((storageNode) => storageNode.address)
+        return stream.getStorageNodes()
     } else {
-        // all storage nodes (currently there is only one)
-        const nodes = await client.getNodes()
-        return nodes.map((n) => n.address)
+        return client.getAllStorageNodes()
     }
 }
 
-const program = new Command()
-program
+createClientCommand(async (client: StreamrClient, options: any) => {
+    const streamId = options.stream
+    const addresses = await getStorageNodes(streamId, client)
+    if (addresses.length > 0) {
+        console.info(EasyTable.print(addresses.map((address: string) => ({
+            address
+        }))))
+    }
+})
     .description('fetch a list of storage nodes')
     .option('-s, --stream <streamId>', 'only storage nodes which store the given stream (needs authentication)')
-
-authOptions(program)
-
-envOptions(program)
-    .version(pkg.version)
-    .action((options: any) => {
-        const client = new StreamrClient(formStreamrOptionsWithEnv(options))
-        const streamId = getStreamId(options.stream, options)
-        getStorageNodes(streamId, client).then((addresses: string[]) => {
-            if (addresses.length > 0) {
-                console.info(EasyTable.print(addresses.map((address: string) => ({
-                    address
-                }))))
-            }
-            return true
-        }).catch((err) => {
-            console.error(err)
-            process.exit(1)
-        })
-    })
-    .parse(process.argv)
-
-exitWithHelpIfArgsNotBetween(program, 0, 0)
+    .parseAsync()

package/bin/streamr-storage-node-remove-stream.ts

@@ -1,32 +1,12 @@
 #!/usr/bin/env node
-import { Command } from 'commander'
-import { StreamrClient, Stream } from 'streamr-client'
-import {
-    envOptions,
-    authOptions,
-    exitWithHelpIfArgsNotBetween,
-    formStreamrOptionsWithEnv,
-    getStreamId,
-} from './common'
-import pkg from '../package.json'
+import '../src/logLevel'
+import StreamrClient from 'streamr-client'
+import { createClientCommand } from '../src/command'
 
-const program = new Command()
-program
+createClientCommand(async (client: StreamrClient, storageNodeAddress: string, streamId: string) => {
+    const stream = await client.getStream(streamId)
+    await stream.removeFromStorageNode(storageNodeAddress)
+})
     .arguments('<storageNodeAddress> <streamId>')
     .description('remove stream from a storage node')
-authOptions(program)
-envOptions(program)
-    .version(pkg.version)
-    .action((storageNodeAddress: string, streamIdOrPath: string, options: any) => {
-        const client = new StreamrClient(formStreamrOptionsWithEnv(options))
-        const streamId = getStreamId(streamIdOrPath, options)!
-        client.getStream(streamId)
-            .then((stream: Stream) => stream.removeFromStorageNode(storageNodeAddress))
-            .catch((err) => {
-                console.error(err)
-                process.exit(1)
-            })
-    })
-    .parse(process.argv)
-
-exitWithHelpIfArgsNotBetween(program, 2, 2)
+    .parseAsync()

package/bin/streamr-storage-node.ts

@@ -10,4 +10,4 @@ program
     .command('add-stream', 'add stream')
     .command('remove-stream', 'remove stream')
     .command('list-stream-parts', 'list stream parts in a storage node')
-    .parse(process.argv)
+    .parse()

package/bin/streamr-stream-create.ts

@@ -1,35 +1,23 @@
 #!/usr/bin/env node
-import { Command } from 'commander'
-import { create } from '../src/create'
-import {
-    envOptions,
-    authOptions,
-    exitWithHelpIfArgsNotBetween,
-    formStreamrOptionsWithEnv,
-    createFnParseInt
-} from './common'
-import pkg from '../package.json'
+import '../src/logLevel'
+import StreamrClient from 'streamr-client'
+import { createFnParseInt } from '../src/common'
+import { createClientCommand } from '../src/command'
 
-const program = new Command()
-program
+createClientCommand(async (client: StreamrClient, streamIdOrPath: string, options: any) => {
+    const body: any = {
+        id: streamIdOrPath,
+        description: options.description,
+        config: options.streamConfig,
+        partitions: options.partitions
+    }
+    const stream = await client.createStream(body)
+    console.info(JSON.stringify(stream.toObject(), null, 2))
+})
     .arguments('<streamId>')
     .description('create a new stream')
     .option('-d, --description <description>', 'define a description')
-    .option('-c, --config <config>', 'define a configuration as JSON', (s: string) => JSON.parse(s))
+    .option('-c, --stream-config <config>', 'define a configuration as JSON', (s: string) => JSON.parse(s))
     .option('-p, --partitions <count>', 'define a partition count',
         createFnParseInt('--partitions'))
-authOptions(program)
-envOptions(program)
-    .version(pkg.version)
-    .action((streamIdOrPath: string, options: any) => {
-        const body: any = {
-            id: streamIdOrPath,
-            description: options.description,
-            config: options.config,
-            partitions: options.partitions
-        }
-        create(body, formStreamrOptionsWithEnv(options))
-    })
-    .parse(process.argv)
-
-exitWithHelpIfArgsNotBetween(program, 1, 1)
+    .parseAsync()

package/bin/streamr-stream-grant-permission.ts

@@ -1,69 +1,10 @@
 #!/usr/bin/env node
-import { Command } from 'commander'
-import {
-    envOptions,
-    authOptions,
-    formStreamrOptionsWithEnv,
-    getStreamId
-} from './common'
-import pkg from '../package.json'
-import { AnonymousStreamPermisson, StreamOperation, StreamrClient, UserStreamPermission } from 'streamr-client'
-import EasyTable from 'easy-table'
-
-const PUBLIC_PERMISSION_ID = 'public'
-const OPERATION_PREFIX = 'stream_'
-
-const getOperation = (id: string) => {
-    // we support both short ids (e.g. "publish"), and long ids (e.g. "stream_publish")
-    // the actual StreamOperation constant is the long id string
-    // backend does the validation of invalid constants
-    if (!id.startsWith(OPERATION_PREFIX)) {
-        return (OPERATION_PREFIX + id) as StreamOperation
-    } else {
-        return id as StreamOperation
-    }
-}
-
-const getShortOperationId = (operation: StreamOperation) => {
-    const longOperationId = operation as string
-    if (longOperationId.startsWith(OPERATION_PREFIX)) {
-        return longOperationId.substring(OPERATION_PREFIX.length)
-    } else {
-        throw new Error(`Assertion failed: unknown prefix for in ${longOperationId}`)
-    }
-}
-
-const getTarget = (user: string): string|undefined => {
-    if (user === PUBLIC_PERMISSION_ID) {
-        return undefined
-    } else {
-        return user
-    }
-}
-
-const program = new Command()
-program
-    .arguments('<streamId> <user> <operations...>')
-    .description('grant permission: use keyword "public" as a user to grant a public permission')
-authOptions(program)
-envOptions(program)
-    .version(pkg.version)
-    .action(async (streamIdOrPath: string, user: string, operationIds: string[], options: any) => {
-        const operations = operationIds.map((o: string) => getOperation(o))
-        const target = getTarget(user)
-        const client = new StreamrClient(formStreamrOptionsWithEnv(options))
-        const streamId = getStreamId(streamIdOrPath, options)!
-        const stream = await client.getStream(streamId)
-        const tasks = operations.map((operation: StreamOperation) => stream.grantPermission(operation, target))
-        const permissions = await Promise.all(tasks)
-        console.info(EasyTable.print(permissions.map((permission: UserStreamPermission|AnonymousStreamPermisson) => ({
-            id: permission.id,
-            operation: getShortOperationId(permission.operation),
-            user: (permission as AnonymousStreamPermisson).anonymous ? PUBLIC_PERMISSION_ID : (permission as UserStreamPermission).user
-        }))))
-    })
-    .parseAsync(process.argv)
-    .catch((e) => {
-        console.error(e)
-        process.exit(1)
-    })
+import '../src/logLevel'
+import { Stream, StreamPermission } from 'streamr-client'
+import { runModifyPermissionsCommand } from '../src/permission'
+
+runModifyPermissionsCommand(
+    (stream: Stream, permission: StreamPermission, target: string) => stream.grantUserPermission(permission, target),
+    (stream: Stream, permission: StreamPermission) => stream.grantPublicPermission(permission),
+    'grant'
+)

package/bin/streamr-stream-publish.ts

@@ -1,30 +1,62 @@
 #!/usr/bin/env node
-import { Command } from 'commander'
+import '../src/logLevel'
+import { Writable } from 'stream'
+import { StreamrClient } from 'streamr-client'
+import { wait } from 'streamr-test-utils'
 import es from 'event-stream'
-import { publishStream } from '../src/publish'
-import { envOptions, authOptions, exitWithHelpIfArgsNotBetween, formStreamrOptionsWithEnv, getStreamId } from './common'
-import pkg from '../package.json'
+import { createClientCommand } from '../src/command'
 
-const program = new Command()
-program
-    .arguments('<streamId>')
-    .description('publish to a stream by reading JSON messages from stdin line-by-line')
-    .option('-k, --partition-key <string>', 'field name in each message to use for assigning the message to a stream partition')
-authOptions(program)
-envOptions(program)
-    .version(pkg.version)
-    .action((streamIdOrPath: string, options: any) => {
-        const streamId = getStreamId(streamIdOrPath, options)!
-        const ps = publishStream(streamId, options.partitionKey, formStreamrOptionsWithEnv(options))
+const publishStream = (
+    stream: string,
+    partitionKeyField: string | undefined,
+    client: StreamrClient
+): Writable => {
+    const writable = new Writable({
+        objectMode: true,
+        write: (data: any, _: any, done: any) => {
+            let json = null
+            // ignore newlines, etc
+            if (!data || String(data).trim() === '') {
+                done()
+                return
+            }
+            try {
+                json = JSON.parse(data)
+            } catch (e) {
+                console.error(data.toString())
+                done(e)
+                return
+            }
+            const partitionKey = (partitionKeyField !== undefined) ? json[partitionKeyField] : undefined
+            client.publish(stream, json, Date.now(), partitionKey).then(
+                () => done(),
+                (err) => done(err)
+            )
+        }
+    })
+    return writable
+}
+
+createClientCommand(async (client: StreamrClient, streamId: string, options: any) => {
+    const ps = publishStream(streamId, options.partitionKeyField, client)
+    return new Promise((resolve, reject) => {
         process.stdin
             .pipe(es.split())
             .pipe(ps)
-            .on('error', (err: any) => {
-                console.error(err)
-                process.exit(1)
-                // process.stdin.pipe(ps) recover pipe to continue execution
+            .once('finish', async () => {
+                // We need to wait some time because the client.publish() may resolve the promise
+                // before the node has propagated the message. That may happend if the node
+                // has not yet connected to Tracker when client.publish() is called. In that case
+                // the message is put to the propagation queue (activeTaskStore.add call in
+                // network Propagation.ts:59) and the client.publish() promise resolves immeditatelly.
+                // TODO Remove this wait when NET-604 has been resolved
+                await wait(2000)
+                resolve(undefined)
             })
+            .once('error', (err: any) => reject(err) )
     })
-    .parse(process.argv)
-
-exitWithHelpIfArgsNotBetween(program, 1, 1)
+})
+    .arguments('<streamId>')
+    .description('publish to a stream by reading JSON messages from stdin line-by-line')
+    .option('-k, --partition-key-field <string>', 'field name in each message to use for assigning the message to a stream partition')
+    .parseAsync()

package/bin/streamr-stream-resend-from.ts

@@ -0,0 +1,26 @@
+#!/usr/bin/env node
+import '../src/logLevel'
+import StreamrClient from 'streamr-client'
+import { createClientCommand } from '../src/command'
+import { resend } from '../src/resend'
+
+createClientCommand(async (client: StreamrClient, from: string, streamId: string, options: any) => {
+    const resendOptions = {
+        from: {
+            timestamp: Date.parse(from),
+            sequenceNumber: 0
+        },
+        publisherId: options.publisherId
+    }
+    await resend(streamId, resendOptions, client, options.subscribe)
+}, {
+    clientOptionsFactory: (options) => ({
+        orderMessages: !options.disableOrdering
+    })
+})
+    .arguments('<from> <streamId>')
+    .description('request messages starting from given date-time (format: "YYYY-MM-DDTHH:mm:ss.sssZ")')
+    .option('--publisher-id <string>', 'filter results by publisher')
+    .option('-d, --disable-ordering', 'disable ordering of messages by OrderingUtil', false)
+    .option('-s, --subscribe', 'subscribe in addition to resend', false)
+    .parseAsync()

package/bin/streamr-stream-resend-last.ts

@@ -0,0 +1,25 @@
+#!/usr/bin/env node
+import '../src/logLevel'
+import StreamrClient from 'streamr-client'
+import { createClientCommand } from '../src/command'
+import { resend } from '../src/resend'
+
+createClientCommand(async (client: StreamrClient, n: string, streamId: string, options: any) => {
+    if (isNaN(n as any)) {
+        console.error('argument n is not a number')
+        process.exit(1)
+    }
+    const resendOptions = {
+        last: parseInt(n)
+    }
+    await resend(streamId, resendOptions, client, options.subscribe)
+}, {
+    clientOptionsFactory: (options) => ({
+        orderMessages: !options.disableOrdering
+    })
+})
+    .arguments('<n> <streamId>')
+    .description('request last N messages')
+    .option('-d, --disable-ordering', 'disable ordering of messages by OrderingUtil', false)
+    .option('-s, --subscribe', 'subscribe in addition to resend', false)
+    .parseAsync()