dolphindb 2.0.903 → 2.0.905
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/browser.js +6 -2
- package/browser.js.map +1 -1
- package/docs.en.json +946 -59
- package/docs.zh.json +56 -57
- package/i18n/dict.json +6 -0
- package/index.d.ts +1 -1
- package/index.js +166 -3
- package/index.js.map +1 -1
- package/package.json +5 -5
package/docs.en.json
CHANGED
|
@@ -4282,6 +4282,181 @@
|
|
|
4282
4282
|
}
|
|
4283
4283
|
]
|
|
4284
4284
|
},
|
|
4285
|
+
"backupDB": {
|
|
4286
|
+
"title": "backupDB",
|
|
4287
|
+
"type": "function",
|
|
4288
|
+
"children": [
|
|
4289
|
+
{
|
|
4290
|
+
"title": "Syntax",
|
|
4291
|
+
"type": "grammer",
|
|
4292
|
+
"children": [
|
|
4293
|
+
{
|
|
4294
|
+
"type": "text",
|
|
4295
|
+
"value": [
|
|
4296
|
+
"backupDB(backupDir, dbPath)"
|
|
4297
|
+
]
|
|
4298
|
+
}
|
|
4299
|
+
]
|
|
4300
|
+
},
|
|
4301
|
+
{
|
|
4302
|
+
"title": "Arguments",
|
|
4303
|
+
"type": "parameters",
|
|
4304
|
+
"children": [
|
|
4305
|
+
{
|
|
4306
|
+
"type": "text",
|
|
4307
|
+
"value": [
|
|
4308
|
+
"`backupDir` is a string indicating the directory to save the backup.",
|
|
4309
|
+
"",
|
|
4310
|
+
"`dbPath` is a string indicating the database path."
|
|
4311
|
+
]
|
|
4312
|
+
}
|
|
4313
|
+
]
|
|
4314
|
+
},
|
|
4315
|
+
{
|
|
4316
|
+
"title": "Details",
|
|
4317
|
+
"type": "detail",
|
|
4318
|
+
"children": [
|
|
4319
|
+
{
|
|
4320
|
+
"type": "text",
|
|
4321
|
+
"value": [
|
|
4322
|
+
"Back up the entire database to the specified directory.",
|
|
4323
|
+
"",
|
|
4324
|
+
"The function is equivalent to ``backup(backupDir, dbPath, force=false, sparallel=true, snapshot=true)``. "
|
|
4325
|
+
]
|
|
4326
|
+
}
|
|
4327
|
+
]
|
|
4328
|
+
},
|
|
4329
|
+
{
|
|
4330
|
+
"title": "Examples",
|
|
4331
|
+
"type": "example",
|
|
4332
|
+
"children": [
|
|
4333
|
+
{
|
|
4334
|
+
"type": "code",
|
|
4335
|
+
"language": "console",
|
|
4336
|
+
"value": [
|
|
4337
|
+
"",
|
|
4338
|
+
" $ dbName = \"dfs://compoDB2\"",
|
|
4339
|
+
" $ n=1000",
|
|
4340
|
+
" $ ID=rand(\"a\"+string(1..10), n)",
|
|
4341
|
+
" $ dates=2017.08.07..2017.08.11",
|
|
4342
|
+
" $ date=rand(dates, n)",
|
|
4343
|
+
" $ x=rand(10, n)",
|
|
4344
|
+
" $ t=table(ID, date, x)",
|
|
4345
|
+
" $ db1 = database(, VALUE, 2017.08.07..2017.08.11)",
|
|
4346
|
+
" $ db2 = database(, HASH,[INT, 20])",
|
|
4347
|
+
" $ if(existsDatabase(dbName)){",
|
|
4348
|
+
" $ \tdropDatabase(dbName)",
|
|
4349
|
+
" $ }",
|
|
4350
|
+
" $ db = database(dbName, COMPO,[ db1,db2])",
|
|
4351
|
+
"",
|
|
4352
|
+
" //create 2 tables",
|
|
4353
|
+
" $ pt1 = db.createPartitionedTable(t, `pt1, `date`x).append!(t)",
|
|
4354
|
+
" $ pt2 = db.createPartitionedTable(t, `pt2, `date`x).append!(t)",
|
|
4355
|
+
"",
|
|
4356
|
+
" $ backupDB(backupDir, dbName)",
|
|
4357
|
+
"",
|
|
4358
|
+
""
|
|
4359
|
+
]
|
|
4360
|
+
},
|
|
4361
|
+
{
|
|
4362
|
+
"type": "text",
|
|
4363
|
+
"value": [
|
|
4364
|
+
"Related functions: :doc:`backup`, :doc:`backupTable`, :doc:`../r/restore`, :doc:`../r/restoreDB`, :doc:`../r/restoreTable`, :doc:`../m/migrate`"
|
|
4365
|
+
]
|
|
4366
|
+
}
|
|
4367
|
+
]
|
|
4368
|
+
}
|
|
4369
|
+
]
|
|
4370
|
+
},
|
|
4371
|
+
"backupTable": {
|
|
4372
|
+
"title": "backupTable",
|
|
4373
|
+
"type": "function",
|
|
4374
|
+
"children": [
|
|
4375
|
+
{
|
|
4376
|
+
"title": "Syntax",
|
|
4377
|
+
"type": "grammer",
|
|
4378
|
+
"children": [
|
|
4379
|
+
{
|
|
4380
|
+
"type": "text",
|
|
4381
|
+
"value": [
|
|
4382
|
+
"backupTable(backupDir, dbPath, tableName)"
|
|
4383
|
+
]
|
|
4384
|
+
}
|
|
4385
|
+
]
|
|
4386
|
+
},
|
|
4387
|
+
{
|
|
4388
|
+
"title": "Arguments",
|
|
4389
|
+
"type": "parameters",
|
|
4390
|
+
"children": [
|
|
4391
|
+
{
|
|
4392
|
+
"type": "text",
|
|
4393
|
+
"value": [
|
|
4394
|
+
"`backupDir` is a string indicating the directory to save the backup.",
|
|
4395
|
+
"",
|
|
4396
|
+
"`dbPath` is a string indicating the database path.",
|
|
4397
|
+
"",
|
|
4398
|
+
"`tableName` is a string indicating the table name."
|
|
4399
|
+
]
|
|
4400
|
+
}
|
|
4401
|
+
]
|
|
4402
|
+
},
|
|
4403
|
+
{
|
|
4404
|
+
"title": "Details",
|
|
4405
|
+
"type": "detail",
|
|
4406
|
+
"children": [
|
|
4407
|
+
{
|
|
4408
|
+
"type": "text",
|
|
4409
|
+
"value": [
|
|
4410
|
+
"Back up a table to the specified directory.",
|
|
4411
|
+
"",
|
|
4412
|
+
"The function is equivalent to ``backup(backupDir, dbPath, force=false, parallel=true, snapshot=true, tableName)``. "
|
|
4413
|
+
]
|
|
4414
|
+
}
|
|
4415
|
+
]
|
|
4416
|
+
},
|
|
4417
|
+
{
|
|
4418
|
+
"title": "Examples",
|
|
4419
|
+
"type": "example",
|
|
4420
|
+
"children": [
|
|
4421
|
+
{
|
|
4422
|
+
"type": "code",
|
|
4423
|
+
"language": "console",
|
|
4424
|
+
"value": [
|
|
4425
|
+
"",
|
|
4426
|
+
" $ dbName = \"dfs://compoDB2\"",
|
|
4427
|
+
" $ n=1000",
|
|
4428
|
+
" $ ID=rand(\"a\"+string(1..10), n)",
|
|
4429
|
+
" $ dates=2017.08.07..2017.08.11",
|
|
4430
|
+
" $ date=rand(dates, n)",
|
|
4431
|
+
" $ x=rand(10, n)",
|
|
4432
|
+
" $ t=table(ID, date, x)",
|
|
4433
|
+
" $ db1 = database(, VALUE, 2017.08.07..2017.08.11)",
|
|
4434
|
+
" $ db2 = database(, HASH,[INT, 20])",
|
|
4435
|
+
" $ if(existsDatabase(dbName)){",
|
|
4436
|
+
" $ \tdropDatabase(dbName)",
|
|
4437
|
+
" $ }",
|
|
4438
|
+
" $ db = database(dbName, COMPO,[ db1,db2])",
|
|
4439
|
+
"",
|
|
4440
|
+
" $ //create 2 tables",
|
|
4441
|
+
" $ pt1 = db.createPartitionedTable(t, `pt1, `date`x).append!(t)",
|
|
4442
|
+
" $ pt2 = db.createPartitionedTable(t, `pt2, `date`x).append!(t)",
|
|
4443
|
+
"",
|
|
4444
|
+
" $ backupTable(backupDir,dbName,`pt1)",
|
|
4445
|
+
" 50",
|
|
4446
|
+
"",
|
|
4447
|
+
""
|
|
4448
|
+
]
|
|
4449
|
+
},
|
|
4450
|
+
{
|
|
4451
|
+
"type": "text",
|
|
4452
|
+
"value": [
|
|
4453
|
+
"Related functions: :doc:`backup`, :doc:`backupDB`, :doc:`../r/restore`, :doc:`../r/restoreDB`, :doc:`../r/restoreTable`, :doc:`../m/migrate`"
|
|
4454
|
+
]
|
|
4455
|
+
}
|
|
4456
|
+
]
|
|
4457
|
+
}
|
|
4458
|
+
]
|
|
4459
|
+
},
|
|
4285
4460
|
"bar": {
|
|
4286
4461
|
"title": "bar",
|
|
4287
4462
|
"type": "function",
|
|
@@ -28910,7 +29085,13 @@
|
|
|
28910
29085
|
{
|
|
28911
29086
|
"type": "text",
|
|
28912
29087
|
"value": [
|
|
28913
|
-
"Return a table with information about the backups of a
|
|
29088
|
+
"Return a table with information about the backups of a DFS table. Each row of the table corresponds to a backed-up partition. The table contains the following columns:",
|
|
29089
|
+
"",
|
|
29090
|
+
"- chunkID: the chunk ID",
|
|
29091
|
+
"- chunkPath: the DFS path to database chunks",
|
|
29092
|
+
"- cid: the commit ID",
|
|
29093
|
+
"- rows: the number of records in a chunk",
|
|
29094
|
+
"- updateTime: the last update time"
|
|
28914
29095
|
]
|
|
28915
29096
|
}
|
|
28916
29097
|
]
|
|
@@ -28924,27 +29105,38 @@
|
|
|
28924
29105
|
"language": "console",
|
|
28925
29106
|
"value": [
|
|
28926
29107
|
" ",
|
|
28927
|
-
" $
|
|
29108
|
+
" $ if(existsDatabase(\"dfs://valuedb\")){",
|
|
29109
|
+
" $ dropDatabase(\"dfs://valuedb\")",
|
|
29110
|
+
" $ }",
|
|
29111
|
+
" $ n=3000000",
|
|
29112
|
+
" $ month=take(2000.01M..2000.04M, n);",
|
|
29113
|
+
" $ x=1..n",
|
|
29114
|
+
" $ t=table(month,x);",
|
|
29115
|
+
"",
|
|
29116
|
+
"",
|
|
29117
|
+
" $ db=database(\"dfs://valuedb\", VALUE, 2000.01M..2000.04M)",
|
|
29118
|
+
" $ pt = db.createPartitionedTable(t, `pt, `month);",
|
|
29119
|
+
" $ pt.append!(t);",
|
|
29120
|
+
" $ backup(\"/home/DolphinDB/backup\",\"dfs://valuedb\",tableName=\"pt\");",
|
|
29121
|
+
" $ getBackupList(\"/home/DolphinDB/backup\",\"dfs://valuedb\",\"pt\");",
|
|
29122
|
+
"",
|
|
28928
29123
|
""
|
|
28929
29124
|
]
|
|
28930
29125
|
},
|
|
28931
29126
|
{
|
|
28932
29127
|
"type": "text",
|
|
28933
29128
|
"value": [
|
|
28934
|
-
"
|
|
28935
|
-
"chunkID
|
|
28936
|
-
"
|
|
28937
|
-
"
|
|
28938
|
-
"
|
|
28939
|
-
"
|
|
28940
|
-
"
|
|
28941
|
-
"
|
|
28942
|
-
"
|
|
28943
|
-
"
|
|
28944
|
-
"
|
|
28945
|
-
"76caa8b4-a7c2-4266-9e22-e44956dd5201 dfs://compoDB/20170811/0_50 114",
|
|
28946
|
-
"e4f5fbb3-4a70-4d99-a8a4-2bbeaed04b6e dfs://compoDB/20170811/50_100 116",
|
|
28947
|
-
"===================================== ============================= ======== ",
|
|
29129
|
+
"+--------------------------------------+---------------------------+--------+---------+-------------------------+",
|
|
29130
|
+
"| chunkID | chunkPath | cid | rows | updateTime |",
|
|
29131
|
+
"+======================================+===========================+========+=========+=========================+",
|
|
29132
|
+
"| 0061427c-4b24-e3b6-425c-c0e1553d3c35 | dfs://valuedb/200001M/b39 | 13,348 | 750,000 | 2022.09.21T15:45:50.931 |",
|
|
29133
|
+
"+--------------------------------------+---------------------------+--------+---------+-------------------------+",
|
|
29134
|
+
"| dabbd90d-6001-f8a9-4d3e-8000d96eba68 | dfs://valuedb/200002M/b39 | 13,348 | 750,000 | 2022.09.21T15:45:50.931 |",
|
|
29135
|
+
"+--------------------------------------+---------------------------+--------+---------+-------------------------+",
|
|
29136
|
+
"| f5c259b4-4be3-f385-46d4-1a1a2d224e9d | dfs://valuedb/200003M/b39 | 13,348 | 750,000 | 2022.09.21T15:45:50.931 |",
|
|
29137
|
+
"+--------------------------------------+---------------------------+--------+---------+-------------------------+",
|
|
29138
|
+
"| 6ed58eb9-a2ae-6197-4f81-3186ca1e8b20 | dfs://valuedb/200004M/b39 | 13,348 | 750,000 | 2022.09.21T15:45:50.931 |",
|
|
29139
|
+
"+--------------------------------------+---------------------------+--------+---------+-------------------------+",
|
|
28948
29140
|
" ",
|
|
28949
29141
|
".. meta:: ",
|
|
28950
29142
|
" :keywords: get"
|
|
@@ -28997,7 +29189,13 @@
|
|
|
28997
29189
|
{
|
|
28998
29190
|
"type": "text",
|
|
28999
29191
|
"value": [
|
|
29000
|
-
"Return a dictionary with information about the backup of a partition in a
|
|
29192
|
+
"Return a dictionary with information about the backup of a partition in a DFS table, which contains the following keys:",
|
|
29193
|
+
"",
|
|
29194
|
+
"- schema: the schema of the table",
|
|
29195
|
+
"- dfsPath: the DFS path to database chunks",
|
|
29196
|
+
"- rows: the number of records in a chunk",
|
|
29197
|
+
"- chunkID: the chunk ID",
|
|
29198
|
+
"- cid: the commit ID"
|
|
29001
29199
|
]
|
|
29002
29200
|
}
|
|
29003
29201
|
]
|
|
@@ -29012,16 +29210,16 @@
|
|
|
29012
29210
|
"value": [
|
|
29013
29211
|
" ",
|
|
29014
29212
|
" $ getBackupMeta(\"/home/DolphinDB/backup\",\"dfs://valuedb\", \"/200001M\",\"pt\")",
|
|
29015
|
-
"",
|
|
29016
29213
|
" schema->",
|
|
29017
|
-
" name typeString typeInt comment",
|
|
29018
|
-
" ----- ---------- ------- -------",
|
|
29019
|
-
" month MONTH 7
|
|
29020
|
-
" x INT 4
|
|
29214
|
+
" name typeString typeInt extra comment",
|
|
29215
|
+
" ----- ---------- ------- ----- -------",
|
|
29216
|
+
" month MONTH 7 ",
|
|
29217
|
+
" x INT 4 ",
|
|
29021
29218
|
"",
|
|
29022
|
-
" dfsPath->dfs://valuedb/200001M/
|
|
29023
|
-
"
|
|
29024
|
-
"
|
|
29219
|
+
" dfsPath->dfs://valuedb/200001M/b39",
|
|
29220
|
+
" rows->750000",
|
|
29221
|
+
" chunkID->0061427c-4b24-e3b6-425c-c0e1553d3c35",
|
|
29222
|
+
" cid->13349",
|
|
29025
29223
|
""
|
|
29026
29224
|
]
|
|
29027
29225
|
},
|
|
@@ -29036,6 +29234,102 @@
|
|
|
29036
29234
|
}
|
|
29037
29235
|
]
|
|
29038
29236
|
},
|
|
29237
|
+
"getBackupStatus": {
|
|
29238
|
+
"title": "getBackupStatus",
|
|
29239
|
+
"type": "function",
|
|
29240
|
+
"children": [
|
|
29241
|
+
{
|
|
29242
|
+
"title": "Syntax",
|
|
29243
|
+
"type": "grammer",
|
|
29244
|
+
"children": [
|
|
29245
|
+
{
|
|
29246
|
+
"type": "text",
|
|
29247
|
+
"value": [
|
|
29248
|
+
"getBackupStatus([userName])"
|
|
29249
|
+
]
|
|
29250
|
+
}
|
|
29251
|
+
]
|
|
29252
|
+
},
|
|
29253
|
+
{
|
|
29254
|
+
"title": "Arguments",
|
|
29255
|
+
"type": "parameters",
|
|
29256
|
+
"children": [
|
|
29257
|
+
{
|
|
29258
|
+
"type": "text",
|
|
29259
|
+
"value": [
|
|
29260
|
+
"`userName` is a string indicating the user name. "
|
|
29261
|
+
]
|
|
29262
|
+
}
|
|
29263
|
+
]
|
|
29264
|
+
},
|
|
29265
|
+
{
|
|
29266
|
+
"title": "Details",
|
|
29267
|
+
"type": "detail",
|
|
29268
|
+
"children": [
|
|
29269
|
+
{
|
|
29270
|
+
"type": "text",
|
|
29271
|
+
"value": [
|
|
29272
|
+
"Return the status of backup/restore tasks. Return a table where each row represents the information of a task.",
|
|
29273
|
+
"",
|
|
29274
|
+
"- userName: user name",
|
|
29275
|
+
"- type: backup/restore types.",
|
|
29276
|
+
"- BACKUP_BY_SQL/RESTORE_BY_SQL: backup/restore by SQL statement",
|
|
29277
|
+
"- BACKUP_BY_COPY_FILE/RESTORE_BY_COPY_FILE: backup/restore by copying files",
|
|
29278
|
+
"- startTime: when the task starts",
|
|
29279
|
+
"- dbName: database path",
|
|
29280
|
+
"- tableName: table name",
|
|
29281
|
+
"- totalPartitions: the number of partitions to be backed up/restored",
|
|
29282
|
+
"- completedPartitions: the number of partitions have been backed up/restored",
|
|
29283
|
+
"- percentComplete: the completion percentage",
|
|
29284
|
+
"- endTime: return the end time of a task if it has ended, otherwise return the estimated completion time",
|
|
29285
|
+
"- completed: whether the task is completed. Return 1 if it is completed, otherwise 0.",
|
|
29286
|
+
"",
|
|
29287
|
+
"Note:",
|
|
29288
|
+
"",
|
|
29289
|
+
"- The number of tasks generated for a backup statement is the same as the number of the backup partitions.",
|
|
29290
|
+
"- Administrators can check either the tasks submitted by a specified user by specifying the userName or the tasks submitted by all users.",
|
|
29291
|
+
"- When a non-administrator executes the function, return the status of backup/restore tasks submitted by the current user."
|
|
29292
|
+
]
|
|
29293
|
+
}
|
|
29294
|
+
]
|
|
29295
|
+
},
|
|
29296
|
+
{
|
|
29297
|
+
"title": "Examples",
|
|
29298
|
+
"type": "example",
|
|
29299
|
+
"children": [
|
|
29300
|
+
{
|
|
29301
|
+
"type": "code",
|
|
29302
|
+
"language": "console",
|
|
29303
|
+
"value": [
|
|
29304
|
+
"",
|
|
29305
|
+
" $ getBackupStatus()",
|
|
29306
|
+
" "
|
|
29307
|
+
]
|
|
29308
|
+
},
|
|
29309
|
+
{
|
|
29310
|
+
"type": "text",
|
|
29311
|
+
"value": [
|
|
29312
|
+
"+---------------+---------------------------------------------+---------------+-----------+-----------------+---------------------+-----------------+---------------------------+",
|
|
29313
|
+
"| userName type | startTime | dbName | tableName | totalPartitions | completedPartitions | percentComplete | endTime completed |",
|
|
29314
|
+
"+===============+=============================================+===============+===========+=================+=====================+=================+===========================+",
|
|
29315
|
+
"| u1 | BACKUP_BY_COPY_FILE 2022.09.21T17:18:04.264 | dfs://valuedb | pt | 1 | 1 | 100 | 2022.09.21T17:18:04.269 1 |",
|
|
29316
|
+
"+---------------+---------------------------------------------+---------------+-----------+-----------------+---------------------+-----------------+---------------------------+",
|
|
29317
|
+
"| u1 | BACKUP 2022.09.21T17:13:04.344 | dfs://valuedb | pt | 4 | 4 | 100 | 2022.09.21T17:13:04.413 1 |",
|
|
29318
|
+
"+---------------+---------------------------------------------+---------------+-----------+-----------------+---------------------+-----------------+---------------------------+",
|
|
29319
|
+
"| u1 | BACKUP_BY_COPY_FILE 2022.09.21T17:18:04.264 | dfs://valuedb | pt1 | 1 | 1 | 100 | 2022.09.21T17:18:04.265 1 |",
|
|
29320
|
+
"+---------------+---------------------------------------------+---------------+-----------+-----------------+---------------------+-----------------+---------------------------+",
|
|
29321
|
+
"| admin | BACKUP_BY_COPY_FILE 2022.09.21T16:47:42.798 | dfs://valuedb | pt | 4 | 4 | 100 | 2022.09.21T16:47:42.859 1 |",
|
|
29322
|
+
"+---------------+---------------------------------------------+---------------+-----------+-----------------+---------------------+-----------------+---------------------------+",
|
|
29323
|
+
"| admin | BACKUP_BY_COPY_FILE 2022.09.21T16:37:33.725 | dfs://valuedb | pt | 4 | 4 | 100 | 2022.09.21T16:37:33.790 1 |",
|
|
29324
|
+
"+---------------+---------------------------------------------+---------------+-----------+-----------------+---------------------+-----------------+---------------------------+",
|
|
29325
|
+
"| admin | BACKUP 2022.09.21T15:10:05.016 | dfs://compoDB | pt2 | 10 | 10 | 100 | 2022.09.21T15:10:05.075 1 |",
|
|
29326
|
+
"+---------------+---------------------------------------------+---------------+-----------+-----------------+---------------------+-----------------+---------------------------+"
|
|
29327
|
+
]
|
|
29328
|
+
}
|
|
29329
|
+
]
|
|
29330
|
+
}
|
|
29331
|
+
]
|
|
29332
|
+
},
|
|
29039
29333
|
"getCacheEngineMemSize": {
|
|
29040
29334
|
"title": "getCacheEngineMemSize",
|
|
29041
29335
|
"type": "function",
|
|
@@ -44832,7 +45126,9 @@
|
|
|
44832
45126
|
{
|
|
44833
45127
|
"type": "text",
|
|
44834
45128
|
"value": [
|
|
44835
|
-
"Load the backup of a partition in a distributed table. It must be executed by a logged-in user."
|
|
45129
|
+
"Load the backup of a partition in a distributed table. It must be executed by a logged-in user.",
|
|
45130
|
+
"",
|
|
45131
|
+
"Currently, this function only supports loading a partition backed up with SQL statement (when the parameter *sqlObj* is specified for function :doc:`../b/backup`)."
|
|
44836
45132
|
]
|
|
44837
45133
|
}
|
|
44838
45134
|
]
|
|
@@ -51926,7 +52222,7 @@
|
|
|
51926
52222
|
"",
|
|
51927
52223
|
"`backupTableName` is a string indicating a table name.",
|
|
51928
52224
|
"",
|
|
51929
|
-
"`newDBPath` is a string indicating the new database name. If not specified, the default value is
|
|
52225
|
+
"`newDBPath` is a string indicating the new database name. If not specified, the default value is *backupDBPath*. To specify the parameter, make sure that the storage engine of the backup database is the same as the engine of newDBPath, and the partitionScheme must be the same (except for VALUE). For a VALUE partitioned database, the partitioning scheme of the backup database must be a subset of that of the database to be restored.",
|
|
51930
52226
|
"",
|
|
51931
52227
|
"`newTableName` is a string indicating the new table name. If not specified, the default value is `backupTableName`."
|
|
51932
52228
|
]
|
|
@@ -59092,7 +59388,9 @@
|
|
|
59092
59388
|
{
|
|
59093
59389
|
"type": "text",
|
|
59094
59390
|
"value": [
|
|
59095
|
-
"
|
|
59391
|
+
"If *X* is a vector, return the number of unique elements in *X*.",
|
|
59392
|
+
"",
|
|
59393
|
+
"If *X* is a tuple, elements of each vector at the same position forms a key, and this function returns the number of unique keys."
|
|
59096
59394
|
]
|
|
59097
59395
|
}
|
|
59098
59396
|
]
|
|
@@ -69591,8 +69889,8 @@
|
|
|
69591
69889
|
}
|
|
69592
69890
|
]
|
|
69593
69891
|
},
|
|
69594
|
-
"
|
|
69595
|
-
"title": "
|
|
69892
|
+
"restoreDB": {
|
|
69893
|
+
"title": "restoreDB",
|
|
69596
69894
|
"type": "function",
|
|
69597
69895
|
"children": [
|
|
69598
69896
|
{
|
|
@@ -69602,7 +69900,7 @@
|
|
|
69602
69900
|
{
|
|
69603
69901
|
"type": "text",
|
|
69604
69902
|
"value": [
|
|
69605
|
-
"
|
|
69903
|
+
"restoreDB(backupDir, dbPath, [newDBPath])"
|
|
69606
69904
|
]
|
|
69607
69905
|
}
|
|
69608
69906
|
]
|
|
@@ -69614,7 +69912,11 @@
|
|
|
69614
69912
|
{
|
|
69615
69913
|
"type": "text",
|
|
69616
69914
|
"value": [
|
|
69617
|
-
"
|
|
69915
|
+
"`backupDir` is a string indicating the directory to save the backup.",
|
|
69916
|
+
"",
|
|
69917
|
+
"`dbPath` is a string indicating the database path.",
|
|
69918
|
+
"",
|
|
69919
|
+
"`newDBPath` is is a string indicating the new database name. The default value is *dbPath*."
|
|
69618
69920
|
]
|
|
69619
69921
|
}
|
|
69620
69922
|
]
|
|
@@ -69626,21 +69928,19 @@
|
|
|
69626
69928
|
{
|
|
69627
69929
|
"type": "text",
|
|
69628
69930
|
"value": [
|
|
69629
|
-
"
|
|
69931
|
+
"Restore the backup database. Return a table where each row is the restored database and table name.",
|
|
69630
69932
|
"",
|
|
69631
|
-
"
|
|
69933
|
+
"Similar to function :doc:`../m/migrate`, the function can restore a database, and the difference lies in:",
|
|
69632
69934
|
"",
|
|
69633
|
-
"
|
|
69935
|
+
"- ``migrate`` can restore all databases and tables under a directory, while ``restoreDB`` can only restore a database.",
|
|
69634
69936
|
"",
|
|
69635
|
-
"
|
|
69636
|
-
"name meaning",
|
|
69637
|
-
"================== =================================================",
|
|
69638
|
-
"chunkId the chunk ID",
|
|
69639
|
-
"srcNode alias of source node",
|
|
69640
|
-
"destNode alias of destination node",
|
|
69641
|
-
"================== =================================================",
|
|
69937
|
+
"- If the names of restored database and tables are the same as the originals, the original databases and tables must be deleted before calling ``migrate``, which is not required by function ``restoreDB``.",
|
|
69642
69938
|
"",
|
|
69643
|
-
"
|
|
69939
|
+
"Note:",
|
|
69940
|
+
"",
|
|
69941
|
+
"- This function can only restore a database backed up by copying files (when dbPath is specified for function backup).",
|
|
69942
|
+
"",
|
|
69943
|
+
"- Make sure that the storage engine of the backed-up database is the same as the engine of *newDBPath*, and the *partitionScheme* (except for VALUE) must be the same. For a VALUE partitioned database, the partitioning scheme of the backup database must be a subset of that of the database to be restored."
|
|
69644
69944
|
]
|
|
69645
69945
|
}
|
|
69646
69946
|
]
|
|
@@ -69653,29 +69953,53 @@
|
|
|
69653
69953
|
"type": "code",
|
|
69654
69954
|
"language": "console",
|
|
69655
69955
|
"value": [
|
|
69656
|
-
""
|
|
69657
|
-
" $ restoreDislocatedTablet()",
|
|
69658
|
-
""
|
|
69956
|
+
" "
|
|
69659
69957
|
]
|
|
69660
69958
|
},
|
|
69661
69959
|
{
|
|
69662
69960
|
"type": "text",
|
|
69663
69961
|
"value": [
|
|
69664
|
-
"
|
|
69665
|
-
"
|
|
69666
|
-
"
|
|
69667
|
-
"
|
|
69668
|
-
"
|
|
69669
|
-
"
|
|
69962
|
+
" $ dbName = \"dfs://compoDB2\"",
|
|
69963
|
+
" $ n=1000",
|
|
69964
|
+
" $ ID=rand(\"a\"+string(1..10), n)",
|
|
69965
|
+
" $ dates=2017.08.07..2017.08.11",
|
|
69966
|
+
" $ date=rand(dates, n)",
|
|
69967
|
+
" $ x=rand(10, n)",
|
|
69968
|
+
" $ t=table(ID, date, x)",
|
|
69969
|
+
" $ db1 = database(, VALUE, 2017.08.07..2017.08.11)",
|
|
69970
|
+
" $ db2 = database(, HASH,[INT, 20])",
|
|
69971
|
+
" $ if(existsDatabase(dbName)){",
|
|
69972
|
+
" $ \tdropDatabase(dbName)",
|
|
69973
|
+
" $ }",
|
|
69974
|
+
" $ db = database(dbName, COMPO,[ db1,db2])",
|
|
69975
|
+
"",
|
|
69976
|
+
" //create 2 tables",
|
|
69977
|
+
" $ pt1 = db.createPartitionedTable(t, `pt1, `date`x).append!(t)",
|
|
69978
|
+
" $ pt2 = db.createPartitionedTable(t, `pt2, `date`x).append!(t)",
|
|
69979
|
+
"",
|
|
69980
|
+
" $ backupDB(backupDir, dbName)",
|
|
69981
|
+
"",
|
|
69982
|
+
" $ restoreDB(backupDir, dbName)",
|
|
69983
|
+
"",
|
|
69984
|
+
"+----------------+-----------+",
|
|
69985
|
+
"| dbName | tableName |",
|
|
69986
|
+
"+================+===========+",
|
|
69987
|
+
"| dfs://compoDB2 | pt1 |",
|
|
69988
|
+
"+----------------+-----------+",
|
|
69989
|
+
"| dfs://compoDB2 | pt2 |",
|
|
69990
|
+
"+----------------+-----------+",
|
|
69991
|
+
"",
|
|
69992
|
+
"",
|
|
69993
|
+
"Related functions: :doc:`restore`, :doc:`restoreTable`, :doc:`../m/migrate`, :doc:`../b/backup`, :doc:`../b/backupDB`, :doc:`../b/backupTable`"
|
|
69670
69994
|
]
|
|
69671
69995
|
}
|
|
69672
69996
|
]
|
|
69673
69997
|
}
|
|
69674
69998
|
]
|
|
69675
69999
|
},
|
|
69676
|
-
"
|
|
69677
|
-
"title": "
|
|
69678
|
-
"type": "
|
|
70000
|
+
"restoreDislocatedTablet": {
|
|
70001
|
+
"title": "restoreDislocatedTablet",
|
|
70002
|
+
"type": "function",
|
|
69679
70003
|
"children": [
|
|
69680
70004
|
{
|
|
69681
70005
|
"title": "Syntax",
|
|
@@ -69684,7 +70008,199 @@
|
|
|
69684
70008
|
{
|
|
69685
70009
|
"type": "text",
|
|
69686
70010
|
"value": [
|
|
69687
|
-
"
|
|
70011
|
+
"restoreDislocatedTablet()"
|
|
70012
|
+
]
|
|
70013
|
+
}
|
|
70014
|
+
]
|
|
70015
|
+
},
|
|
70016
|
+
{
|
|
70017
|
+
"title": "Arguments",
|
|
70018
|
+
"type": "parameters",
|
|
70019
|
+
"children": [
|
|
70020
|
+
{
|
|
70021
|
+
"type": "text",
|
|
70022
|
+
"value": [
|
|
70023
|
+
"None"
|
|
70024
|
+
]
|
|
70025
|
+
}
|
|
70026
|
+
]
|
|
70027
|
+
},
|
|
70028
|
+
{
|
|
70029
|
+
"title": "Details",
|
|
70030
|
+
"type": "detail",
|
|
70031
|
+
"children": [
|
|
70032
|
+
{
|
|
70033
|
+
"type": "text",
|
|
70034
|
+
"value": [
|
|
70035
|
+
"When enableChunkGranularityConfig=false (see details in :doc:`/DatabaseandDistributedComputing/Configuration/StandaloneMode` *enableChunkGranularityConfig*) all tables in the same partition are distributed in the same node. ",
|
|
70036
|
+
"",
|
|
70037
|
+
"When rebalancing data by function :doc:`/FunctionsandCommands/FunctionReferences/r/rebalanceChunksAmongDataNodes`, if the node is down, some tables in the same partition may fail to relocate, i.e., different tables under the same partition will be distributed in different nodes. Execute the function to move all tables under the same partition to one node.",
|
|
70038
|
+
"",
|
|
70039
|
+
"Return a table containing the following columns:",
|
|
70040
|
+
"",
|
|
70041
|
+
"================== =================================================",
|
|
70042
|
+
"name meaning",
|
|
70043
|
+
"================== =================================================",
|
|
70044
|
+
"chunkId the chunk ID",
|
|
70045
|
+
"srcNode alias of source node",
|
|
70046
|
+
"destNode alias of destination node",
|
|
70047
|
+
"================== =================================================",
|
|
70048
|
+
"",
|
|
70049
|
+
"You can get the status of recovery tasks by :doc:`/FunctionsandCommands/FunctionReferences/g/getRecoveryTaskStatus` on a controller."
|
|
70050
|
+
]
|
|
70051
|
+
}
|
|
70052
|
+
]
|
|
70053
|
+
},
|
|
70054
|
+
{
|
|
70055
|
+
"title": "Examples",
|
|
70056
|
+
"type": "example",
|
|
70057
|
+
"children": [
|
|
70058
|
+
{
|
|
70059
|
+
"type": "code",
|
|
70060
|
+
"language": "console",
|
|
70061
|
+
"value": [
|
|
70062
|
+
"",
|
|
70063
|
+
" $ restoreDislocatedTablet()",
|
|
70064
|
+
""
|
|
70065
|
+
]
|
|
70066
|
+
},
|
|
70067
|
+
{
|
|
70068
|
+
"type": "text",
|
|
70069
|
+
"value": [
|
|
70070
|
+
"==================================== ======= ========",
|
|
70071
|
+
"ChunkId srcNode destNode",
|
|
70072
|
+
"==================================== ======= ========",
|
|
70073
|
+
"99279094-ca12-3b87-48b6-520cbb986f39 node1 node2",
|
|
70074
|
+
"45f612b8-42f5-aebd-4cef-e522b6ae1fc8 node1 node2\t",
|
|
70075
|
+
"==================================== ======= ========"
|
|
70076
|
+
]
|
|
70077
|
+
}
|
|
70078
|
+
]
|
|
70079
|
+
}
|
|
70080
|
+
]
|
|
70081
|
+
},
|
|
70082
|
+
"restoreTable": {
|
|
70083
|
+
"title": "restoreTable",
|
|
70084
|
+
"type": "function",
|
|
70085
|
+
"children": [
|
|
70086
|
+
{
|
|
70087
|
+
"title": "Syntax",
|
|
70088
|
+
"type": "grammer",
|
|
70089
|
+
"children": [
|
|
70090
|
+
{
|
|
70091
|
+
"type": "text",
|
|
70092
|
+
"value": [
|
|
70093
|
+
"restoreTable(backupDir, dbPath, tableName, [newDBPath], [newTableName])"
|
|
70094
|
+
]
|
|
70095
|
+
}
|
|
70096
|
+
]
|
|
70097
|
+
},
|
|
70098
|
+
{
|
|
70099
|
+
"title": "Arguments",
|
|
70100
|
+
"type": "parameters",
|
|
70101
|
+
"children": [
|
|
70102
|
+
{
|
|
70103
|
+
"type": "text",
|
|
70104
|
+
"value": [
|
|
70105
|
+
"`backupDir` is a string indicating the directory to save the backup.",
|
|
70106
|
+
"",
|
|
70107
|
+
"`dbPath` is a string indicating the database path.",
|
|
70108
|
+
"",
|
|
70109
|
+
"`tableName` is a string indicating the table name.",
|
|
70110
|
+
"",
|
|
70111
|
+
"`newDBPath` is a string indicating the new database name. The default value is dbPath.",
|
|
70112
|
+
"",
|
|
70113
|
+
"`newTableName` is a string indicating the new table name. The default value is *tableName*."
|
|
70114
|
+
]
|
|
70115
|
+
}
|
|
70116
|
+
]
|
|
70117
|
+
},
|
|
70118
|
+
{
|
|
70119
|
+
"title": "Details",
|
|
70120
|
+
"type": "detail",
|
|
70121
|
+
"children": [
|
|
70122
|
+
{
|
|
70123
|
+
"type": "text",
|
|
70124
|
+
"value": [
|
|
70125
|
+
"Restore the backup database. Return a table where each row is the restored database and table name. The function is equivalent to ``restore(backupDir, dbPath, tableName, force=false, parallel=true, snapshot=true)``.",
|
|
70126
|
+
"",
|
|
70127
|
+
"Similar to function :doc:`../m/migrate`, the function can restore all tables of a database, and the difference lies in:",
|
|
70128
|
+
"",
|
|
70129
|
+
"- ``migrate`` can restore all databases and tables under a directory, while ``restoreTable`` can only restore a table.",
|
|
70130
|
+
"",
|
|
70131
|
+
"- When the names of restored database and tables are the same as the originals, the original database and tables must be deleted before calling ``migrate``, which is not required by function ``restoreTable``.",
|
|
70132
|
+
"",
|
|
70133
|
+
"Note:",
|
|
70134
|
+
"",
|
|
70135
|
+
"- This function can only restore a database backed up by copying files (when dbPath is specified for function backup).",
|
|
70136
|
+
"",
|
|
70137
|
+
"- Make sure that the storage engine of the backed-up database is the same as the engine of *newDBPath*, and the partitionScheme (except for VALUE) must be the same. For a VALUE partitioned database, the partitioning scheme of the backup database must be a subset of that of the database to be restored."
|
|
70138
|
+
]
|
|
70139
|
+
}
|
|
70140
|
+
]
|
|
70141
|
+
},
|
|
70142
|
+
{
|
|
70143
|
+
"title": "Examples",
|
|
70144
|
+
"type": "example",
|
|
70145
|
+
"children": [
|
|
70146
|
+
{
|
|
70147
|
+
"type": "code",
|
|
70148
|
+
"language": "console",
|
|
70149
|
+
"value": [
|
|
70150
|
+
" "
|
|
70151
|
+
]
|
|
70152
|
+
},
|
|
70153
|
+
{
|
|
70154
|
+
"type": "text",
|
|
70155
|
+
"value": [
|
|
70156
|
+
" $ dbName = \"dfs://compoDB2\"",
|
|
70157
|
+
" $ n=1000",
|
|
70158
|
+
" $ ID=rand(\"a\"+string(1..10), n)",
|
|
70159
|
+
" $ dates=2017.08.07..2017.08.11",
|
|
70160
|
+
" $ date=rand(dates, n)",
|
|
70161
|
+
" $ x=rand(10, n)",
|
|
70162
|
+
" $ t=table(ID, date, x)",
|
|
70163
|
+
" $ db1 = database(, VALUE, 2017.08.07..2017.08.11)",
|
|
70164
|
+
" $ db2 = database(, HASH,[INT, 20])",
|
|
70165
|
+
" $ if(existsDatabase(dbName)){",
|
|
70166
|
+
" $ \tdropDatabase(dbName)",
|
|
70167
|
+
" $ }",
|
|
70168
|
+
" $ db = database(dbName, COMPO,[ db1,db2])",
|
|
70169
|
+
"",
|
|
70170
|
+
" //create 2 tables",
|
|
70171
|
+
" $ pt1 = db.createPartitionedTable(t, `pt1, `date`x).append!(t)",
|
|
70172
|
+
" $ pt2 = db.createPartitionedTable(t, `pt2, `date`x).append!(t)",
|
|
70173
|
+
"",
|
|
70174
|
+
" $ backupDB(backupDir, dbName)",
|
|
70175
|
+
"",
|
|
70176
|
+
" $ restoreTable(backupDir,\"dfs://compoDB2\",`pt1)",
|
|
70177
|
+
"",
|
|
70178
|
+
"+----------------+-----------+",
|
|
70179
|
+
"| dbName | tableName |",
|
|
70180
|
+
"+================+===========+",
|
|
70181
|
+
"| dfs://compoDB2 | pt1 |",
|
|
70182
|
+
"+----------------+-----------+",
|
|
70183
|
+
"",
|
|
70184
|
+
"",
|
|
70185
|
+
"Related functions: :doc:`restore`, :doc:`restoreDB`, :doc:`../m/migrate`, :doc:`../b/backup`, :doc:`../b/backupDB`, :doc:`../b/backupTable`"
|
|
70186
|
+
]
|
|
70187
|
+
}
|
|
70188
|
+
]
|
|
70189
|
+
}
|
|
70190
|
+
]
|
|
70191
|
+
},
|
|
70192
|
+
"resumeRecovery": {
|
|
70193
|
+
"title": "resumeRecovery",
|
|
70194
|
+
"type": "command",
|
|
70195
|
+
"children": [
|
|
70196
|
+
{
|
|
70197
|
+
"title": "Syntax",
|
|
70198
|
+
"type": "grammer",
|
|
70199
|
+
"children": [
|
|
70200
|
+
{
|
|
70201
|
+
"type": "text",
|
|
70202
|
+
"value": [
|
|
70203
|
+
"resumeRecovery()"
|
|
69688
70204
|
]
|
|
69689
70205
|
}
|
|
69690
70206
|
]
|
|
@@ -81468,6 +81984,377 @@
|
|
|
81468
81984
|
}
|
|
81469
81985
|
]
|
|
81470
81986
|
},
|
|
81987
|
+
"streamFilter": {
|
|
81988
|
+
"title": "streamFilter",
|
|
81989
|
+
"type": "function",
|
|
81990
|
+
"children": [
|
|
81991
|
+
{
|
|
81992
|
+
"title": "Details",
|
|
81993
|
+
"type": "detail",
|
|
81994
|
+
"children": [
|
|
81995
|
+
{
|
|
81996
|
+
"type": "text",
|
|
81997
|
+
"value": [
|
|
81998
|
+
"Create an engine that splits the ingested stream for different handlers. Return a table object.",
|
|
81999
|
+
"",
|
|
82000
|
+
"The engine works as follows:",
|
|
82001
|
+
"",
|
|
82002
|
+
"\\1. Deserialize the ingested data.",
|
|
82003
|
+
"",
|
|
82004
|
+
"Note: This step only takes place when the ingested data is from a heterogeneous stream table, i.e., the output of heterogeneous :doc:`../r/replay`. ",
|
|
82005
|
+
"",
|
|
82006
|
+
"\\2. Split the ingested stream based on the conditions as specified by *filter*.",
|
|
82007
|
+
"",
|
|
82008
|
+
"\\3. Ingest the split streams to the handlers as specified by *filter* in the order of their timestamps.",
|
|
82009
|
+
"",
|
|
82010
|
+
"Note: ",
|
|
82011
|
+
"",
|
|
82012
|
+
"Starting from version 1.30.18/2.00.6, in addition to heterogeneous stream tables, ``streamFilter`` also supports processing data from standard stream tables."
|
|
82013
|
+
]
|
|
82014
|
+
}
|
|
82015
|
+
]
|
|
82016
|
+
},
|
|
82017
|
+
{
|
|
82018
|
+
"title": "Syntax",
|
|
82019
|
+
"type": "grammer",
|
|
82020
|
+
"children": [
|
|
82021
|
+
{
|
|
82022
|
+
"type": "text",
|
|
82023
|
+
"value": [
|
|
82024
|
+
"streamFilter(name, dummyTable, filter, [msgSchema], [timeColumn], [conditionColumn])"
|
|
82025
|
+
]
|
|
82026
|
+
}
|
|
82027
|
+
]
|
|
82028
|
+
},
|
|
82029
|
+
{
|
|
82030
|
+
"title": "Arguments",
|
|
82031
|
+
"type": "parameters",
|
|
82032
|
+
"children": [
|
|
82033
|
+
{
|
|
82034
|
+
"type": "text",
|
|
82035
|
+
"value": [
|
|
82036
|
+
"`name` is a STRING scalar indicating the name of the stream filter engine. It must begin with a letter and may contain letters, numbers and underscores.",
|
|
82037
|
+
"",
|
|
82038
|
+
"`dummyTable` is a table. It has the same schema as the stream table that the stream filter subscribes to. The table can be empty.",
|
|
82039
|
+
"",
|
|
82040
|
+
"`filter` is a dictionary or a tuple of dictionaries. It defines how to process the ingested data. Each dictionary can have the following key-value pairs:",
|
|
82041
|
+
"",
|
|
82042
|
+
"* 'timeRange' [optional] is a pair or a tuple of pairs. Apply it to *timeColumn* to filter for records in the specified time range. Note: When processing a standard stream table, *timeRange* must have the same data type as *timeColumn*.",
|
|
82043
|
+
"* 'condition':",
|
|
82044
|
+
"",
|
|
82045
|
+
" * When processing a heterogeneous stream table: It is a STRING referring to a dictionary key from the *inputTables* of :doc:`../r/replay`. The engine will filter records by the specified key.",
|
|
82046
|
+
" * When processing a standard stream table: It is a STRING scalar/vector indicating the value(s) from the *conditionColumn*, or metacode of one or more Boolean expressions (without any function calls). The engine will filter records by the specified condition.",
|
|
82047
|
+
"",
|
|
82048
|
+
"* 'handler' is a unary function or a table (can be the table object returned by a streaming engine).",
|
|
82049
|
+
"",
|
|
82050
|
+
" * If it's a function, the filter result (a table) is passed as the function's sole argument.",
|
|
82051
|
+
" * If it's a table object, the filtered data are inserted into the table directly.",
|
|
82052
|
+
"",
|
|
82053
|
+
"`msgSchema` [optional] A dictionary",
|
|
82054
|
+
"",
|
|
82055
|
+
"* When processing a heterogeneous stream table: The dictionary indicates the input tables of :doc:`../r/replay`. The keys are the table identifiers as specified in the *inputTables* parameter of :doc:`../r/replay` and the values indicate the schema of each table. The ingested data will be parsed based on *msgSchema*.",
|
|
82056
|
+
"* When processing a standard stream table: Do not specify the parameter.",
|
|
82057
|
+
"",
|
|
82058
|
+
"The following parameters are only required when processing data from *a standard stream table*:",
|
|
82059
|
+
"",
|
|
82060
|
+
"`timeColumn` is a STRING indicating the name of the temporal column in *dummyTable*. If unspecified, it takes the name of the first column in the *dummyTable*.",
|
|
82061
|
+
"",
|
|
82062
|
+
"`conditionColumn` is a STRING indicating a column (must be STRING or SYMBOL type) in *dummyTable*. If this parameter is unspecified, the \"condition\" key of filter takes no effect.",
|
|
82063
|
+
""
|
|
82064
|
+
]
|
|
82065
|
+
}
|
|
82066
|
+
]
|
|
82067
|
+
},
|
|
82068
|
+
{
|
|
82069
|
+
"title": "Examples",
|
|
82070
|
+
"type": "example",
|
|
82071
|
+
"children": [
|
|
82072
|
+
{
|
|
82073
|
+
"type": "text",
|
|
82074
|
+
"value": [
|
|
82075
|
+
"\\(1) Processing a heterogeneous stream table:",
|
|
82076
|
+
"",
|
|
82077
|
+
"Replay the DFS tables \"orders\" and \"trades\" to simulate an asof join of the two streams. ",
|
|
82078
|
+
"",
|
|
82079
|
+
"If we simply perform an N-to-N replay on the two tables, it is not guaranteed that the records will be ingested to the left and right tables of the asof join engine in chronological order: It may happen that a record with a larger timestamp arrives in the left table before a record with a smaller timestamp arrives in the right table. For more information, see :doc:`../r/replay`.",
|
|
82080
|
+
"",
|
|
82081
|
+
"Therefore, we replay the two tables into one heterogeneous stream table to make sure all records are ordered by timestamp. The stream filter subscribes to the heterogenous stream table, splits the ingested data into two streams and distributes them to the left and right tables of the asof join engine (:doc:`../c/createAsofJoinEngine`). In this way, we can make sure that the data is ingested to the left and right tables of the asof join engine in the order of the timestamps.",
|
|
82082
|
+
""
|
|
82083
|
+
]
|
|
82084
|
+
},
|
|
82085
|
+
{
|
|
82086
|
+
"type": "code",
|
|
82087
|
+
"language": "console",
|
|
82088
|
+
"value": [
|
|
82089
|
+
""
|
|
82090
|
+
]
|
|
82091
|
+
},
|
|
82092
|
+
{
|
|
82093
|
+
"type": "text",
|
|
82094
|
+
"value": [
|
|
82095
|
+
" //create the \"orders\" table",
|
|
82096
|
+
" $ n=1000",
|
|
82097
|
+
" $ sym = take(take(\"IBM\",n).join(take(\"GS\",n)), n*2*3)",
|
|
82098
|
+
" $ date=take(2022.01.04..2022.01.06, n*2*3).sort!()",
|
|
82099
|
+
" $ timestamp1=take(2022.01.04 09:30:00.000+rand(1000,n),n) join take(2022.01.04 09:31:00.000+rand(1000,n),n)",
|
|
82100
|
+
" $ timestamp2=take(2022.01.05 09:30:00.000+rand(1000,n),n) join take(2022.01.05 09:31:00.000+rand(1000,n),n)",
|
|
82101
|
+
" $ timestamp3=take(2022.01.06 09:30:00.000+rand(1000,n),n) join take(2022.01.06 09:31:00.000+rand(1000,n),n)",
|
|
82102
|
+
" $ timestamp=timestamp1 join timestamp2 join timestamp3",
|
|
82103
|
+
" $ volume = rand(100, n*2*3)",
|
|
82104
|
+
" $ t=table(sym,date,timestamp,volume)",
|
|
82105
|
+
"",
|
|
82106
|
+
" $ if(existsDatabase(\"dfs://test_order\")){",
|
|
82107
|
+
" $ dropDatabase(\"dfs://test_order\")",
|
|
82108
|
+
" $ }",
|
|
82109
|
+
" $ db1_or=database(\"\",RANGE, 2022.01.04..2022.01.07)",
|
|
82110
|
+
" $ db2_or=database(\"\",VALUE,`IBM`GS)",
|
|
82111
|
+
" $ db_or=database(\"dfs://test_order\",COMPO,[db1_or, db2_or])",
|
|
82112
|
+
" $ orders=db_or.createPartitionedTable(t,`orders,`date`sym)",
|
|
82113
|
+
" $ orders.append!(t);",
|
|
82114
|
+
" $ select count(*) from orders",
|
|
82115
|
+
" 6000",
|
|
82116
|
+
"",
|
|
82117
|
+
" //create the \"trades\" table",
|
|
82118
|
+
" $ n=2000 ",
|
|
82119
|
+
" $ sym = take(take(\"IBM\",n).join(take(\"GS\",n)), n*2*3)",
|
|
82120
|
+
" $ date=take(2022.01.04..2022.01.06, n*2*3).sort!()",
|
|
82121
|
+
" $ timestamp1=take(2022.01.04 09:30:00.000+rand(1000,n),n) join take(2022.01.04 09:31:00.000+rand(1000,n),n)",
|
|
82122
|
+
" $ timestamp2=take(2022.01.05 09:30:00.000+rand(1000,n),n) join take(2022.01.05 09:31:00.000+rand(1000,n),n)",
|
|
82123
|
+
" $ timestamp3=take(2022.01.06 09:30:00.000+rand(1000,n),n) join take(2022.01.06 09:31:00.000+rand(1000,n),n)",
|
|
82124
|
+
" $ timestamp=timestamp1 join timestamp2 join timestamp3",
|
|
82125
|
+
" $ volume = rand(100, n*2*3)",
|
|
82126
|
+
" $ price = rand(50.0, n*3) join rand(20.0, n*3)",
|
|
82127
|
+
"",
|
|
82128
|
+
" $ t=table(sym,date,timestamp,volume,price)",
|
|
82129
|
+
"",
|
|
82130
|
+
" $ if(existsDatabase(\"dfs://test_trades\")){",
|
|
82131
|
+
" $ dropDatabase(\"dfs://test_trades\")",
|
|
82132
|
+
" $ }",
|
|
82133
|
+
" $ db1=database(\"\",RANGE, 2022.01.04..2022.01.07)",
|
|
82134
|
+
" $ db2=database(\"\",VALUE,`IBM`GS)",
|
|
82135
|
+
" $ db=database(\"dfs://test_trades\",COMPO,[db1, db2])",
|
|
82136
|
+
" $ trades=db.createPartitionedTable(t,`trades,`date`sym)",
|
|
82137
|
+
" $ trades.append!(t);",
|
|
82138
|
+
" $ select count(*) from trades",
|
|
82139
|
+
" 12000",
|
|
82140
|
+
"",
|
|
82141
|
+
" //generate the heterogeneous data sources and create a table as the the outputTable of replay()",
|
|
82142
|
+
" $ ds_or = replayDS(sqlObj=<select * from loadTable(db_or, `orders)>, dateColumn=`date, timeColumn=`timestamp)",
|
|
82143
|
+
" $ ds = replayDS(sqlObj=<select * from loadTable(db, `trades)>, dateColumn=`date, timeColumn=`timestamp)",
|
|
82144
|
+
" $ input_dict=dict([\"orders\",\"trades\"], [ds_or, ds])",
|
|
82145
|
+
" $ share streamTable(100:0,`timestamp`sym`blob`volume, [TIMESTAMP,SYMBOL, BLOB, INT]) as opt",
|
|
82146
|
+
"",
|
|
82147
|
+
"",
|
|
82148
|
+
" //subscribe to the output table of replay to ingest the data to the stream filter",
|
|
82149
|
+
" $ share streamTable(100:0,`timestamp`sym`blob`volume, [TIMESTAMP,SYMBOL, BLOB, INT]) as streamFilterOpt",
|
|
82150
|
+
" $ share streamTable(100:0, `sym`date`timestamp`volume, [SYMBOL, DATE, TIMESTAMP, INT] ) as streamOrders",
|
|
82151
|
+
" $ share streamTable(100:0, `sym`date`timestamp`volume`price, [SYMBOL, DATE, TIMESTAMP, INT, DOUBLE] ) as streamTrades",
|
|
82152
|
+
" $ streamOpt=table(100:0, `timestamp`sym`volume`price`result, [TIMESTAMP, SYMBOL, INT, DOUBLE, DOUBLE])",
|
|
82153
|
+
"",
|
|
82154
|
+
" $ filter1=dict(STRING,ANY)",
|
|
82155
|
+
" $ filter1['condition']=`orders",
|
|
82156
|
+
" $ filter1['timeRange']=09:30:00.000:09:30:00.005",
|
|
82157
|
+
"",
|
|
82158
|
+
" $ filter2=dict(STRING,ANY)",
|
|
82159
|
+
" $ filter2['condition']=`trades",
|
|
82160
|
+
" $ filter2['timeRange']=09:30:00.000:09:30:00.005",
|
|
82161
|
+
"",
|
|
82162
|
+
" $ ajEngine=createAsofJoinEngine(name=\"ajEngine\", leftTable=streamOrders, rightTable=streamTrades, outputTable=streamOpt, metrics=<[volume,price,price*volume]>, matchingColumn=`sym, useSystemTime=true)",
|
|
82163
|
+
" $ filter1['handler']=getLeftStream(ajEngine)",
|
|
82164
|
+
" $ filter2['handler']=getRightStream(ajEngine)",
|
|
82165
|
+
" $ schema=dict([\"orders\",\"trades\"], [streamOrders, streamTrades])",
|
|
82166
|
+
"",
|
|
82167
|
+
" $ engine=streamFilter(name=`streamFilter,dummyTable=streamFilterOpt, filter=[filter1,filter2],msgSchema=schema) ",
|
|
82168
|
+
" $ subscribeTable(tableName=\"opt\", actionName=\"sub1\", offset=0, handler=engine, msgAsTable=true)",
|
|
82169
|
+
"",
|
|
82170
|
+
" //replay the heterogeneous data sources and output to the table \"opt\"",
|
|
82171
|
+
" $ replay(inputTables=input_dict,outputTables=opt, timeColumn=`timestamp)",
|
|
82172
|
+
"",
|
|
82173
|
+
" $ select count(*) from streamOpt",
|
|
82174
|
+
" 20",
|
|
82175
|
+
"",
|
|
82176
|
+
" //drop the subscription",
|
|
82177
|
+
" unsubscribeTable(tableName=\"opt\", actionName=\"sub1\")",
|
|
82178
|
+
" dropStreamEngine(`streamFilter)",
|
|
82179
|
+
" dropStreamEngine(`ajEngine)",
|
|
82180
|
+
"",
|
|
82181
|
+
"",
|
|
82182
|
+
"\\(2) Processing standard stream table:",
|
|
82183
|
+
"",
|
|
82184
|
+
"In this example, data from the standard stream table “trades“ is ingested to the stream filter, where the records are filtered and assigned to handlers for further processing.",
|
|
82185
|
+
""
|
|
82186
|
+
]
|
|
82187
|
+
},
|
|
82188
|
+
{
|
|
82189
|
+
"type": "code",
|
|
82190
|
+
"language": "console",
|
|
82191
|
+
"value": [
|
|
82192
|
+
""
|
|
82193
|
+
]
|
|
82194
|
+
},
|
|
82195
|
+
{
|
|
82196
|
+
"type": "text",
|
|
82197
|
+
"value": [
|
|
82198
|
+
" $ n=20",
|
|
82199
|
+
" $ sym = symbol(take(`A`B`C,n))",
|
|
82200
|
+
" $ name = string(rand(1..10,n))",
|
|
82201
|
+
" $ date = temporalAdd(2012.12.06,0..(n-1),'d')",
|
|
82202
|
+
" $ time = temporalAdd(09:30:00.000,0..(n-1),'ms')",
|
|
82203
|
+
" $ vol = 100+take(1..8,20)",
|
|
82204
|
+
" $ t = table(date,time,sym,name,vol)",
|
|
82205
|
+
"",
|
|
82206
|
+
" $ share streamTable(100:0,`date`time`sym`name`vol,[DATE,TIME,SYMBOL,STRING,INT]) as st1",
|
|
82207
|
+
" $ share streamTable(100:0,`date`time`sym`name`vol,[DATE,TIME,SYMBOL,STRING,INT]) as st2",
|
|
82208
|
+
" $ share streamTable(100:0,`date`time`sym`name`vol,[DATE,TIME,SYMBOL,STRING,INT]) as st3",
|
|
82209
|
+
"",
|
|
82210
|
+
"",
|
|
82211
|
+
" $ share streamTable(100:0,`time`sym`sum_vol,[TIME,SYMBOL,INT]) as output1",
|
|
82212
|
+
" $ share streamTable(100:0,`time`avg_vol,[TIME,INT]) as output2",
|
|
82213
|
+
"",
|
|
82214
|
+
" // create 2 streaming engines has the handlers of the stream filter",
|
|
82215
|
+
" $ engine1=createTimeSeriesEngine(name=\"timeEngine\", windowSize=3, step=3, metrics=<[sum(vol)]>, dummyTable=st3, outputTable=output1, timeColumn=`time, useSystemTime=false, keyColumn=`sym, garbageSize=50)",
|
|
82216
|
+
" $ engine2=createReactiveStateEngine(name=\"reactiveEngine\", metrics=<[mavg(vol, 3)]>, dummyTable=st1, outputTable=output2, keyColumn=`sym)",
|
|
82217
|
+
"",
|
|
82218
|
+
" //share \"trades\" as the stream table to be subscribed by the stream filter",
|
|
82219
|
+
" $ share streamTable(100:0,`date`time`sym`name`vol,[DATE,TIME,SYMBOL,STRING,INT]) as trades",
|
|
82220
|
+
"",
|
|
82221
|
+
" //set the first filter and ingest the result to engine2",
|
|
82222
|
+
" $ filter1 = dict(STRING,ANY)",
|
|
82223
|
+
" $ filter1['condition']=`A",
|
|
82224
|
+
" $ filter1['handler']=engine2",
|
|
82225
|
+
" $ filter1['timeRange']=(09:30:00.001:09:30:00.010,09:29:00.000:09:30:00.000)",
|
|
82226
|
+
"",
|
|
82227
|
+
" //set the second filter and ingest the result to st2",
|
|
82228
|
+
" $ filter2 = dict(STRING,ANY)",
|
|
82229
|
+
" $ filter2['handler']=st2",
|
|
82230
|
+
" $ filter2['timeRange']=09:30:00.002:09:30:00.005",
|
|
82231
|
+
"",
|
|
82232
|
+
" //set the first filter and ingest the result to engine1",
|
|
82233
|
+
" $ filter3 = dict(STRING,ANY)",
|
|
82234
|
+
" $ filter3['condition']=`C`A",
|
|
82235
|
+
" $ filter3['handler']=engine1",
|
|
82236
|
+
"",
|
|
82237
|
+
" ///The stream filter subscribes to the stream table \"trades\" and distributes the ingested data based on the specified conditions",
|
|
82238
|
+
" $ streamFilter2=streamFilter(name=\"streamFilterDemo\",dummyTable=trades,filter=[filter1,filter2,filter3], timeColumn=`time, conditionColumn=`sym)",
|
|
82239
|
+
" $ subscribeTable(tableName=\"trades\", actionName=\"sub1\", offset=0, handler=streamFilter2, msgAsTable=true)",
|
|
82240
|
+
" $ trades.append!(t)",
|
|
82241
|
+
""
|
|
82242
|
+
]
|
|
82243
|
+
},
|
|
82244
|
+
{
|
|
82245
|
+
"type": "code",
|
|
82246
|
+
"language": "console",
|
|
82247
|
+
"value": [
|
|
82248
|
+
""
|
|
82249
|
+
]
|
|
82250
|
+
},
|
|
82251
|
+
{
|
|
82252
|
+
"type": "text",
|
|
82253
|
+
"value": [
|
|
82254
|
+
" $ select * from output1",
|
|
82255
|
+
"",
|
|
82256
|
+
"============ ===== ========",
|
|
82257
|
+
"time sym sum_vol",
|
|
82258
|
+
"============ ===== ========",
|
|
82259
|
+
"09:30:00.003 A 101",
|
|
82260
|
+
"09:30:00.003 C 103",
|
|
82261
|
+
"09:30:00.006 A 104",
|
|
82262
|
+
"09:30:00.006 C 106",
|
|
82263
|
+
"09:30:00.009 A 107",
|
|
82264
|
+
"09:30:00.009 C 101",
|
|
82265
|
+
"09:30:00.012 A 102",
|
|
82266
|
+
"09:30:00.012 C 104",
|
|
82267
|
+
"09:30:00.015 A 105",
|
|
82268
|
+
"09:30:00.015 C 107",
|
|
82269
|
+
"09:30:00.018 A 108",
|
|
82270
|
+
"============ ===== ========",
|
|
82271
|
+
""
|
|
82272
|
+
]
|
|
82273
|
+
},
|
|
82274
|
+
{
|
|
82275
|
+
"type": "code",
|
|
82276
|
+
"language": "console",
|
|
82277
|
+
"value": [
|
|
82278
|
+
""
|
|
82279
|
+
]
|
|
82280
|
+
},
|
|
82281
|
+
{
|
|
82282
|
+
"type": "text",
|
|
82283
|
+
"value": [
|
|
82284
|
+
" $ select * from output2",
|
|
82285
|
+
"",
|
|
82286
|
+
"+-------------+--------+",
|
|
82287
|
+
"|time |avg_vol |",
|
|
82288
|
+
"+=============+========+",
|
|
82289
|
+
"|00:00:00.001 | |",
|
|
82290
|
+
"+-------------+--------+",
|
|
82291
|
+
"|00:00:00.001 | |",
|
|
82292
|
+
"+-------------+--------+",
|
|
82293
|
+
"|00:00:00.001 | 104 |",
|
|
82294
|
+
"+-------------+--------+",
|
|
82295
|
+
"|00:00:00.001 | 104 |",
|
|
82296
|
+
"+-------------+--------+",
|
|
82297
|
+
""
|
|
82298
|
+
]
|
|
82299
|
+
},
|
|
82300
|
+
{
|
|
82301
|
+
"type": "code",
|
|
82302
|
+
"language": "console",
|
|
82303
|
+
"value": [
|
|
82304
|
+
""
|
|
82305
|
+
]
|
|
82306
|
+
},
|
|
82307
|
+
{
|
|
82308
|
+
"type": "text",
|
|
82309
|
+
"value": [
|
|
82310
|
+
" $ select * from st2",
|
|
82311
|
+
"",
|
|
82312
|
+
"============ ============= === ==== ====",
|
|
82313
|
+
"date time sym name vol",
|
|
82314
|
+
"============ ============= === ==== ====",
|
|
82315
|
+
"2012.12.08 09:30:00.002 C 6 103",
|
|
82316
|
+
"2012.12.09 09:30:00.003 A 8 104",
|
|
82317
|
+
"2012.12.10 09:30:00.004 B 10 105",
|
|
82318
|
+
"2012.12.11 09:30:00.005 C 10 106",
|
|
82319
|
+
"2012.12.12 09:30:00.006 A 10 107",
|
|
82320
|
+
"2012.12.13 09:30:00.007 B 1 108",
|
|
82321
|
+
"2012.12.14 09:30:00.008 C 3 101",
|
|
82322
|
+
"2012.12.15 09:30:00.009 A 4 102",
|
|
82323
|
+
"2012.12.16 09:30:00.010 B 9 103",
|
|
82324
|
+
"============ ============= === ==== ====",
|
|
82325
|
+
"",
|
|
82326
|
+
"\"condition\" can also be specified as Boolean expressions to support more complex filter logic. In the example above, replace the value of \"condition\" in filter2 with a Boolean expression to filter the data based on the columns \"vol\" and “date“.",
|
|
82327
|
+
""
|
|
82328
|
+
]
|
|
82329
|
+
},
|
|
82330
|
+
{
|
|
82331
|
+
"type": "code",
|
|
82332
|
+
"language": "console",
|
|
82333
|
+
"value": [
|
|
82334
|
+
""
|
|
82335
|
+
]
|
|
82336
|
+
},
|
|
82337
|
+
{
|
|
82338
|
+
"type": "text",
|
|
82339
|
+
"value": [
|
|
82340
|
+
" $ filter2 = dict(STRING,ANY)",
|
|
82341
|
+
" $ filter2['condition'] = <sym==`A and 101<vol<105 and date<2012.12.15>",
|
|
82342
|
+
" $ filter2['handler'] = st2",
|
|
82343
|
+
" $ filter2['timeRange'] = 09:30:00.002:09:30:00.010",
|
|
82344
|
+
" ",
|
|
82345
|
+
" $ select * from st2",
|
|
82346
|
+
"",
|
|
82347
|
+
"=========== ============ === ==== ====",
|
|
82348
|
+
"date time sym name\tvol",
|
|
82349
|
+
"=========== ============ === ==== ====",
|
|
82350
|
+
"2012.12.09 09:30:00.003 A 7 104",
|
|
82351
|
+
"=========== ============ === ==== ===="
|
|
82352
|
+
]
|
|
82353
|
+
}
|
|
82354
|
+
]
|
|
82355
|
+
}
|
|
82356
|
+
]
|
|
82357
|
+
},
|
|
81471
82358
|
"streamTable": {
|
|
81472
82359
|
"title": "streamTable",
|
|
81473
82360
|
"type": "function",
|
|
@@ -82263,7 +83150,7 @@
|
|
|
82263
83150
|
"",
|
|
82264
83151
|
"`jobDesc` is a string indicating job description.",
|
|
82265
83152
|
"",
|
|
82266
|
-
"`
|
|
83153
|
+
"`jobDef` is a local function that defines the job. Please note that it is not the function name, and therefore it should not be quoted.",
|
|
82267
83154
|
"",
|
|
82268
83155
|
"`args...` is the arguments of the function. If the function has no arguments, it is unspecified."
|
|
82269
83156
|
]
|
|
@@ -82478,7 +83365,7 @@
|
|
|
82478
83365
|
{
|
|
82479
83366
|
"type": "text",
|
|
82480
83367
|
"value": [
|
|
82481
|
-
"`
|
|
83368
|
+
"`jobId` is a string indicating the job ID.",
|
|
82482
83369
|
"",
|
|
82483
83370
|
"`jobDesc` is a string of the job description.",
|
|
82484
83371
|
"",
|
|
@@ -82486,7 +83373,7 @@
|
|
|
82486
83373
|
"",
|
|
82487
83374
|
"`parallelism` is a positive integer indicating the maximum number of workers allocated to the job. ",
|
|
82488
83375
|
"",
|
|
82489
|
-
"`
|
|
83376
|
+
"`jobDef` is a local function that defines the job. Please note that it is not the function name, and therefore it should not be quoted.",
|
|
82490
83377
|
"",
|
|
82491
83378
|
"`args...` is the arguments of the local function. If the function has no arguments, it is unspecified."
|
|
82492
83379
|
]
|