@ethernity-cloud/sdk-js 1.7.14 → 1.7.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -2
- package/cas/config.js +12 -1
- package/nodenithy/build/securelock/src/ecld_state.js +127 -18
- package/nodenithy/run/etny-securelock-test.yaml.tpl +3 -0
- package/nodenithy/run.js +2 -1
- package/package.json +3 -2
- package/pynithy/build/securelock/src/ecld_state.py +117 -20
- package/pynithy/run/etny-securelock-test.yaml.tpl +3 -0
- package/pynithy/run.js +2 -1
- package/run.js +9 -31
- package/runReport.js +45 -0
- package/test.js +1 -1
package/README.md
CHANGED
|
@@ -44,9 +44,9 @@ The sdk has been tested on the following operating systems:
|
|
|
44
44
|
`ecld-publish` registers the image in the ImageRegistry
|
|
45
45
|
(ECImageRegistryV3, `0xa372a6e1...`) before anything discloses its
|
|
46
46
|
name, which makes the name the publishing wallet's; then registers the
|
|
47
|
-
securelock session on-chain in the SessionRegistry (`
|
|
47
|
+
securelock session on-chain in the SessionRegistry (`0x78f0c5E9...`, see
|
|
48
48
|
`cas/config.js`), which takes it only from that wallet, resolves a CAS
|
|
49
|
-
from the ValidatorRegistry (`
|
|
49
|
+
from the ValidatorRegistry (`0x8CC8B1dc...`, or `ECLD_CAS_ADDR=host:port`)
|
|
50
50
|
for the compose and the certificate harvest, takes the trustedzone
|
|
51
51
|
session name from the ImageRegistry, and links the session to the
|
|
52
52
|
published image. `ecld-build` signs the securelock `--production`.
|
|
@@ -182,6 +182,11 @@ sets the enclave heap (default `1024M`).
|
|
|
182
182
|
`--network`/`--securelock`/`--trustedzone`; the signing key is `PRIVATE_KEY`
|
|
183
183
|
(or `ECLD_PRIVATE_KEY`) — a funded `0x` key that pays for the order. Resources
|
|
184
184
|
are tunable with `--task-price`/`--cpu`/`--memory`/`--storage`/`--bandwidth`/`--duration`/`--validators`.
|
|
185
|
+
`--input <file>` / `--input-text <string>` send the task's input dataset, read
|
|
186
|
+
as `___etny_data_set___`; it travels encrypted, and the trustedzone checks it
|
|
187
|
+
against the sha256 the request carries. Exit code 0 on a `SUCCESS` task result,
|
|
188
|
+
1 otherwise; a task that raised in the enclave prints its task code (e.g.
|
|
189
|
+
`4 (BASE_EXCEPTION)`) and the enclave's error report as its result.
|
|
185
190
|
`--ipfs intake` (the default) delivers the payload to the bootnode's payload
|
|
186
191
|
intake once the request is on chain; `--ipfs <url>` writes and reads through
|
|
187
192
|
that IPFS RPC API instead, with `ECLD_IPFS_TOKEN` (or `IPFS_TOKEN`) as its
|
package/cas/config.js
CHANGED
|
@@ -120,8 +120,19 @@ const renderCompose = (content, cas, unsafe) => content
|
|
|
120
120
|
.map((line) => line.replace(/\s*# __(?:CAS|NO_CAS|SAFE)_ONLY__$/, ''))
|
|
121
121
|
.join('\n');
|
|
122
122
|
|
|
123
|
+
// The securelock session's ESR environment entry. A CAS-provisioned enclave
|
|
124
|
+
// takes its environment from the session, not from the image's ENV, so the
|
|
125
|
+
// registry address the build baked must also be set here; without ESR (or
|
|
126
|
+
// without an address) the entry is a YAML comment.
|
|
127
|
+
const esrSessionEnvironment = (env) => {
|
|
128
|
+
const enabled = /^(1|true|yes)$/i.test(String(env.ESR_ENABLED || '').trim());
|
|
129
|
+
const address = String(env.ESR_CONTRACT_ADDRESS || '').trim();
|
|
130
|
+
if (!enabled || !/^0x[0-9a-fA-F]{40}$/.test(address)) return '# ESR not enabled';
|
|
131
|
+
return `ESR_CONTRACT_ADDRESS: "${address}"`;
|
|
132
|
+
};
|
|
133
|
+
|
|
123
134
|
module.exports = {
|
|
124
135
|
VALIDATOR_REGISTRY, SESSION_REGISTRY, CHAIN, casProvisioned, isMainnetNetwork,
|
|
125
136
|
UNSAFE_NETWORKS, isUnsafeNetwork, TESTNETS, isTestnetNetwork, sessionAttestation,
|
|
126
|
-
nameOnNetwork, sessionTag, renderCompose,
|
|
137
|
+
nameOnNetwork, sessionTag, renderCompose, esrSessionEnvironment,
|
|
127
138
|
};
|
|
@@ -72,6 +72,7 @@ function configure({ identityPriv, swiftStreamService, bucket, contractAddress,
|
|
|
72
72
|
_provider = provider;
|
|
73
73
|
_esrLedger = [];
|
|
74
74
|
_taskNonces = {};
|
|
75
|
+
_taskWrites = {};
|
|
75
76
|
_taskRelayNonces = {};
|
|
76
77
|
_taskLedger = {};
|
|
77
78
|
_taskCaller = normAddr(caller);
|
|
@@ -116,6 +117,13 @@ class StateNonceError extends Error {}
|
|
|
116
117
|
// task code 38 (ESR_COMMIT_LIMIT_EXCEEDED).
|
|
117
118
|
const MAX_COMMITS_PER_TASK = 256;
|
|
118
119
|
|
|
120
|
+
// How long a read waits for the node to stage a state object the chain names
|
|
121
|
+
// before the task fails, and how often it looks for it: the node writes the
|
|
122
|
+
// objects pinned on it within seconds of the order's start and the ones it has
|
|
123
|
+
// to fetch from the network as they arrive.
|
|
124
|
+
const STATE_STAGE_WAIT_MS = 120000;
|
|
125
|
+
const STATE_STAGE_POLL_MS = 2000;
|
|
126
|
+
|
|
119
127
|
class StateLimitError extends Error {}
|
|
120
128
|
|
|
121
129
|
function unwrapStored(stored) {
|
|
@@ -260,15 +268,86 @@ function stateKey() {
|
|
|
260
268
|
return keccak256(Buffer.concat([Buffer.from(DOMAIN_SEP_STATE, 'utf8'), material]));
|
|
261
269
|
}
|
|
262
270
|
|
|
271
|
+
// A state object is stored as the node's Kubo stores it with `add
|
|
272
|
+
// --cid-version=1 --raw-leaves` and its default chunker: in raw sha256 blocks
|
|
273
|
+
// of BLOCK_SIZE bytes (the last one shorter), a single block as it is, several
|
|
274
|
+
// under a balanced UnixFS tree of at most LINKS_PER_NODE blocks per node.
|
|
275
|
+
// unixfsCid() computes the resulting CID from the bytes, so the enclave
|
|
276
|
+
// commits a CID it derived itself rather than one a node handed it, and the
|
|
277
|
+
// node stores the object under that CID (mvp-pox-node utils.py unixfs_cid).
|
|
278
|
+
const BLOCK_SIZE = 262144;
|
|
279
|
+
const LINKS_PER_NODE = 174;
|
|
280
|
+
|
|
281
|
+
function cidB32(cid) {
|
|
282
|
+
return 'b' + base32Encode(cid).toLowerCase().replace(/=+$/, '');
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
function pbVarint(n) {
|
|
286
|
+
const out = [];
|
|
287
|
+
for (;;) {
|
|
288
|
+
const byte = n % 128;
|
|
289
|
+
n = Math.floor(n / 128);
|
|
290
|
+
if (n === 0) {
|
|
291
|
+
out.push(byte);
|
|
292
|
+
return Buffer.from(out);
|
|
293
|
+
}
|
|
294
|
+
out.push(byte | 0x80);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
// A protobuf field: varint (wire 0) or length-delimited (wire 2).
|
|
299
|
+
function pbField(number, wire, payload) {
|
|
300
|
+
const tag = pbVarint(number * 8 + wire);
|
|
301
|
+
return wire === 2 ? Buffer.concat([tag, pbVarint(payload.length), payload]) : Buffer.concat([tag, payload]);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// CIDv1, codec raw (0x55), sha2-256 (0x12, 32 bytes).
|
|
305
|
+
function pbRawCid(content) {
|
|
306
|
+
return Buffer.concat([Buffer.from([0x01, 0x55, 0x12, 0x20]), crypto.createHash('sha256').update(content).digest()]);
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
// A dag-pb UnixFS File node over `children`, each [cid, tsize, filesize];
|
|
310
|
+
// returns [cid, tsize, filesize]. Links come before Data in dag-pb; a link
|
|
311
|
+
// carries the child's CID, an empty Name and its Tsize, the size of the
|
|
312
|
+
// child's subtree; Data is UnixFS Type=File(2), filesize, blocksizes.
|
|
313
|
+
function pbFileNode(children) {
|
|
314
|
+
const links = Buffer.concat(children.map(([cid, tsize]) =>
|
|
315
|
+
pbField(2, 2, Buffer.concat([pbField(1, 2, cid), pbField(2, 2, Buffer.alloc(0)), pbField(3, 0, pbVarint(tsize))]))));
|
|
316
|
+
const filesize = children.reduce((n, [, , size]) => n + size, 0);
|
|
317
|
+
const data = Buffer.concat([pbField(1, 0, pbVarint(2)), pbField(3, 0, pbVarint(filesize)),
|
|
318
|
+
...children.map(([, , size]) => pbField(4, 0, pbVarint(size)))]);
|
|
319
|
+
const block = Buffer.concat([links, pbField(1, 2, data)]);
|
|
320
|
+
const cid = Buffer.concat([Buffer.from([0x01, 0x70, 0x12, 0x20]), crypto.createHash('sha256').update(block).digest()]);
|
|
321
|
+
return [cid, block.length + children.reduce((n, [, tsize]) => n + tsize, 0), filesize];
|
|
322
|
+
}
|
|
323
|
+
|
|
263
324
|
/**
|
|
264
|
-
*
|
|
265
|
-
*
|
|
266
|
-
*
|
|
325
|
+
* The CID the node's Kubo gives `content`: the raw block's CID up to one
|
|
326
|
+
* block, else the root of the balanced UnixFS tree over the blocks,
|
|
327
|
+
* LINKS_PER_NODE to a node, every node full but the last.
|
|
328
|
+
*/
|
|
329
|
+
function unixfsCid(content, blockSize = BLOCK_SIZE) {
|
|
330
|
+
const buf = Buffer.isBuffer(content) ? content : Buffer.from(content);
|
|
331
|
+
if (buf.length <= blockSize) return cidB32(pbRawCid(buf));
|
|
332
|
+
let nodes = [];
|
|
333
|
+
for (let i = 0; i < buf.length; i += blockSize) {
|
|
334
|
+
const chunk = buf.subarray(i, i + blockSize);
|
|
335
|
+
nodes.push([pbRawCid(chunk), chunk.length, chunk.length]);
|
|
336
|
+
}
|
|
337
|
+
while (nodes.length > 1) {
|
|
338
|
+
const next = [];
|
|
339
|
+
for (let i = 0; i < nodes.length; i += LINKS_PER_NODE) next.push(pbFileNode(nodes.slice(i, i + LINKS_PER_NODE)));
|
|
340
|
+
nodes = next;
|
|
341
|
+
}
|
|
342
|
+
return cidB32(nodes[0][0]);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/**
|
|
346
|
+
* CIDv1/raw/sha2-256 for `content`: its CID as one raw block, which is its CID
|
|
347
|
+
* through the node's Kubo up to BLOCK_SIZE bytes.
|
|
267
348
|
*/
|
|
268
349
|
function cidv1Raw(content) {
|
|
269
|
-
|
|
270
|
-
const raw = Buffer.concat([Buffer.from([0x01, 0x55, 0x12, 0x20]), digest]);
|
|
271
|
-
return 'b' + base32Encode(raw).toLowerCase().replace(/=+$/, '');
|
|
350
|
+
return cidB32(pbRawCid(content));
|
|
272
351
|
}
|
|
273
352
|
|
|
274
353
|
// RFC4648 base32, no padding stripped here (the caller strips it).
|
|
@@ -356,6 +435,12 @@ let _esrLedger = [];
|
|
|
356
435
|
// even before the node's relay lands on-chain.
|
|
357
436
|
let _taskNonces = {};
|
|
358
437
|
|
|
438
|
+
// Per-task memo of this task's last commit per key, [acl, data, version, cid]
|
|
439
|
+
// as the chain shows them once the node's relay lands. A read prefers it while
|
|
440
|
+
// the chain still shows an older version, so get() after commit() returns the
|
|
441
|
+
// committed state and a second commit to the key builds on the first.
|
|
442
|
+
let _taskWrites = {};
|
|
443
|
+
|
|
359
444
|
// Per-task memo of the NEXT relay nonce per key hash, so multiple commits to
|
|
360
445
|
// the same key within one task sign sequential relay nonces even while the
|
|
361
446
|
// node is still relaying the earlier ones.
|
|
@@ -429,6 +514,10 @@ class StateRegistry {
|
|
|
429
514
|
/** [acl, data, version, cid] for `key`; NO permission check here. */
|
|
430
515
|
async _readContainer(key, fallback = {}) {
|
|
431
516
|
const [cid, version] = await this._contract().getState(this.walletAddress, keyHash(key));
|
|
517
|
+
const own = _taskWrites[key];
|
|
518
|
+
if (own && own[2] > Number(version || 0)) {
|
|
519
|
+
return own;
|
|
520
|
+
}
|
|
432
521
|
if (!version || !cid) {
|
|
433
522
|
return [null, fallback, 0, null];
|
|
434
523
|
}
|
|
@@ -579,7 +668,7 @@ class StateRegistry {
|
|
|
579
668
|
|
|
580
669
|
const blob = encrypt(Buffer.from(JSON.stringify(stored), 'utf8'));
|
|
581
670
|
// Compute the CID from OUR bytes — never accept one from the node.
|
|
582
|
-
const cid =
|
|
671
|
+
const cid = unixfsCid(blob);
|
|
583
672
|
|
|
584
673
|
// Hand the blob and the CID to the node for pinning. It verifies the CID
|
|
585
674
|
// matches; it does not get to choose it. Fire-and-forget.
|
|
@@ -597,6 +686,7 @@ class StateRegistry {
|
|
|
597
686
|
// results): this is what the chain shows once the relay lands.
|
|
598
687
|
ledgerRecord(key, currentVersion + 1, cid, newData);
|
|
599
688
|
_taskNonces[key] = finalNonce;
|
|
689
|
+
_taskWrites[key] = [nextAcl, newData, currentVersion + 1, cid];
|
|
600
690
|
return [nextAcl, newData];
|
|
601
691
|
} catch (e) {
|
|
602
692
|
const msg = String(e && e.message ? e.message : e);
|
|
@@ -623,22 +713,40 @@ class StateRegistry {
|
|
|
623
713
|
* the node stages state objects in the bucket. That makes the node the
|
|
624
714
|
* delivery path but not a trusted one: content is checked against the CID the
|
|
625
715
|
* chain records, so a substituted blob is rejected rather than decrypted.
|
|
716
|
+
*
|
|
717
|
+
* The node stages it as state.<cid>.enc, within seconds of the order's start
|
|
718
|
+
* when the object is pinned on it and as it arrives when it has to fetch it
|
|
719
|
+
* from the network, so the object is looked for every STATE_STAGE_POLL_MS
|
|
720
|
+
* for up to STATE_STAGE_WAIT_MS. state.<key>.enc is the last blob a commit
|
|
721
|
+
* of this key published, which matches only while no later commit was
|
|
722
|
+
* published; one that does not match is passed over.
|
|
626
723
|
*/
|
|
627
724
|
async _fetch(key, cid) {
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
);
|
|
725
|
+
const deadline = Date.now() + STATE_STAGE_WAIT_MS;
|
|
726
|
+
let mismatched = [];
|
|
727
|
+
for (;;) {
|
|
728
|
+
mismatched = [];
|
|
729
|
+
for (const name of [`state.${cid}.enc`, `state.${key}.enc`]) {
|
|
730
|
+
// eslint-disable-next-line no-await-in-loop
|
|
731
|
+
const data = await _swift.getFileContentBytes(_bucket, name).catch(() => null);
|
|
732
|
+
if (data && data.length) {
|
|
733
|
+
const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
|
734
|
+
if (unixfsCid(buf) === cid) {
|
|
735
|
+
return buf;
|
|
736
|
+
}
|
|
737
|
+
mismatched.push(name);
|
|
637
738
|
}
|
|
638
|
-
return buf;
|
|
639
739
|
}
|
|
740
|
+
if (Date.now() >= deadline) break;
|
|
741
|
+
// eslint-disable-next-line no-await-in-loop
|
|
742
|
+
await new Promise((resolve) => { setTimeout(resolve, STATE_STAGE_POLL_MS); });
|
|
743
|
+
}
|
|
744
|
+
if (mismatched.length) {
|
|
745
|
+
throw new Error(
|
|
746
|
+
`State object ${mismatched.join(' and ')} does not match the committed CID ${cid} — refusing to use it`
|
|
747
|
+
);
|
|
640
748
|
}
|
|
641
|
-
throw new Error(`Could not read state object for '${key}' (${cid})`);
|
|
749
|
+
throw new Error(`Could not read state object for '${key}' (${cid}): not staged within ${STATE_STAGE_WAIT_MS / 1000} s`);
|
|
642
750
|
}
|
|
643
751
|
|
|
644
752
|
async _sendCommit(keyHashHex, cid, expectedVersion, ownerAddr = null, nonce = 0) {
|
|
@@ -815,6 +923,7 @@ module.exports = {
|
|
|
815
923
|
StatePermissionError,
|
|
816
924
|
StateNonceError,
|
|
817
925
|
StateLimitError,
|
|
926
|
+
unixfsCid,
|
|
818
927
|
esrGrant,
|
|
819
928
|
esrRevoke,
|
|
820
929
|
esrSetPublicRead,
|
|
@@ -20,6 +20,9 @@ services:
|
|
|
20
20
|
pwd: /
|
|
21
21
|
environment:
|
|
22
22
|
GREETING: hello from ETNY SECURELOCK!!!!
|
|
23
|
+
# A CAS-provisioned enclave takes its environment from this session,
|
|
24
|
+
# not from the image's ENV; run.js fills in the ESR registry address.
|
|
25
|
+
__ESR_ENVIRONMENT__
|
|
23
26
|
|
|
24
27
|
images:
|
|
25
28
|
- name: application_image
|
package/nodenithy/run.js
CHANGED
|
@@ -311,7 +311,8 @@ const main = async () => {
|
|
|
311
311
|
const replacementsSecurelock = {
|
|
312
312
|
PREDECESSOR: PREDECESSOR_HASH_SECURELOCK === 'EMPTY' ? `# predecessor: ${PREDECESSOR_HASH_SECURELOCK}` : `predecessor: ${PREDECESSOR_HASH_SECURELOCK}`,
|
|
313
313
|
MRENCLAVE: process.env.MRENCLAVE_SECURELOCK,
|
|
314
|
-
ENCLAVE_NAME: ENCLAVE_NAME_SECURELOCK
|
|
314
|
+
ENCLAVE_NAME: ENCLAVE_NAME_SECURELOCK,
|
|
315
|
+
ESR_ENVIRONMENT: casConfig.esrSessionEnvironment(process.env)
|
|
315
316
|
};
|
|
316
317
|
if (cas) {
|
|
317
318
|
processYamlTemplate('etny-securelock-test.yaml.tpl', 'etny-securelock-test.yaml', replacementsSecurelock);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ethernity-cloud/sdk-js",
|
|
3
|
-
"version": "1.7.
|
|
3
|
+
"version": "1.7.17",
|
|
4
4
|
"description": "A package to run ethernity cloud build scripts",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"test": "node --test \"test/**/*.test.js\"",
|
|
@@ -32,6 +32,7 @@
|
|
|
32
32
|
"test.js",
|
|
33
33
|
"info.js",
|
|
34
34
|
"run.js",
|
|
35
|
+
"runReport.js",
|
|
35
36
|
"network.js",
|
|
36
37
|
"localKubo.js",
|
|
37
38
|
"revert.js",
|
|
@@ -39,7 +40,7 @@
|
|
|
39
40
|
"!**/*.pyc"
|
|
40
41
|
],
|
|
41
42
|
"dependencies": {
|
|
42
|
-
"@ethernity-cloud/runner": "^0.6.
|
|
43
|
+
"@ethernity-cloud/runner": "^0.6.12",
|
|
43
44
|
"acorn": "^8.11.0",
|
|
44
45
|
"adm-zip": "^0.5.14",
|
|
45
46
|
"axios": "^1.7.2",
|
|
@@ -46,6 +46,7 @@ import hashlib
|
|
|
46
46
|
import io
|
|
47
47
|
import json
|
|
48
48
|
import os
|
|
49
|
+
import time
|
|
49
50
|
|
|
50
51
|
DOMAIN_SEP_STATE = b"ethernity-cloud/esr-encryption/v1"
|
|
51
52
|
|
|
@@ -75,6 +76,11 @@ _task_ledger = {}
|
|
|
75
76
|
# get_nonce() right after a commit in the SAME task returns the fresh value
|
|
76
77
|
# even though the node's relay (and thus the chain) has not landed yet.
|
|
77
78
|
_task_nonces = {}
|
|
79
|
+
# Per-task memo of this task's last commit per key, (acl, data, version, cid)
|
|
80
|
+
# as the chain shows them once the node's relay lands. A read prefers it while
|
|
81
|
+
# the chain still shows an older version, so get() after commit() returns the
|
|
82
|
+
# committed state and a second commit to the key builds on the first.
|
|
83
|
+
_task_writes = {}
|
|
78
84
|
# Per-task memo of the NEXT relay nonce per key hash (hex), so multiple
|
|
79
85
|
# commits to the same key within one task sign sequential relay nonces even
|
|
80
86
|
# while the node is still relaying the earlier ones.
|
|
@@ -312,7 +318,7 @@ def configure(identity_priv, swift_stream_service, bucket, contract_address,
|
|
|
312
318
|
value from the client payload or the node's .env.
|
|
313
319
|
"""
|
|
314
320
|
global _identity_priv, _swift, _bucket, _contract_address, _web3, _esr_evidence
|
|
315
|
-
global _task_ledger, _task_nonces, _task_relay_nonces, _task_caller
|
|
321
|
+
global _task_ledger, _task_nonces, _task_writes, _task_relay_nonces, _task_caller
|
|
316
322
|
_identity_priv = identity_priv
|
|
317
323
|
_swift = swift_stream_service
|
|
318
324
|
_bucket = bucket
|
|
@@ -320,6 +326,7 @@ def configure(identity_priv, swift_stream_service, bucket, contract_address,
|
|
|
320
326
|
_web3 = web3
|
|
321
327
|
_task_ledger = {}
|
|
322
328
|
_task_nonces = {}
|
|
329
|
+
_task_writes = {}
|
|
323
330
|
_task_relay_nonces = {}
|
|
324
331
|
_esr_evidence = []
|
|
325
332
|
_task_caller = _norm_addr(caller)
|
|
@@ -357,6 +364,13 @@ class StatePermissionError(RuntimeError):
|
|
|
357
364
|
# 0x00 = 1 commit ... 0xFF = 256; an all-zero root means no commits at all).
|
|
358
365
|
MAX_COMMITS_PER_TASK = 256
|
|
359
366
|
|
|
367
|
+
# How long a read waits for the node to stage a state object the chain names
|
|
368
|
+
# before the task fails, and how often it looks for it: the node writes the
|
|
369
|
+
# objects pinned on it within seconds of the order's start and the ones it
|
|
370
|
+
# has to fetch from the network as they arrive.
|
|
371
|
+
STATE_STAGE_WAIT_SECONDS = 120
|
|
372
|
+
STATE_STAGE_POLL_SECONDS = 2
|
|
373
|
+
|
|
360
374
|
|
|
361
375
|
class StateLimitError(RuntimeError):
|
|
362
376
|
"""Raised when a task tries to make more than MAX_COMMITS_PER_TASK state
|
|
@@ -478,15 +492,77 @@ def looks_like_cid(value) -> bool:
|
|
|
478
492
|
return False
|
|
479
493
|
|
|
480
494
|
|
|
481
|
-
|
|
482
|
-
|
|
495
|
+
# A state object is stored as the node's Kubo stores it with `add
|
|
496
|
+
# --cid-version=1 --raw-leaves` and its default chunker: in raw sha256 blocks
|
|
497
|
+
# of BLOCK_SIZE bytes (the last one shorter), a single block as it is, several
|
|
498
|
+
# under a balanced UnixFS tree of at most LINKS_PER_NODE blocks per node.
|
|
499
|
+
# unixfs_cid() computes the resulting CID from the bytes, so the enclave
|
|
500
|
+
# commits a CID it derived itself rather than one a node handed it, and the
|
|
501
|
+
# node stores the object under that CID (mvp-pox-node utils.py unixfs_cid).
|
|
502
|
+
BLOCK_SIZE = 262144
|
|
503
|
+
LINKS_PER_NODE = 174
|
|
483
504
|
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
"""
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
505
|
+
|
|
506
|
+
def _cid_b32(cid: bytes) -> str:
|
|
507
|
+
return "b" + base64.b32encode(cid).decode("ascii").lower().rstrip("=")
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def _pb_varint(n: int) -> bytes:
|
|
511
|
+
out = bytearray()
|
|
512
|
+
while True:
|
|
513
|
+
byte = n & 0x7F
|
|
514
|
+
n >>= 7
|
|
515
|
+
if n:
|
|
516
|
+
out.append(byte | 0x80)
|
|
517
|
+
else:
|
|
518
|
+
out.append(byte)
|
|
519
|
+
return bytes(out)
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def _pb_field(number: int, wire: int, payload: bytes) -> bytes:
|
|
523
|
+
"""A protobuf field: varint (wire 0) or length-delimited (wire 2)."""
|
|
524
|
+
tag = _pb_varint((number << 3) | wire)
|
|
525
|
+
return tag + (_pb_varint(len(payload)) + payload if wire == 2 else payload)
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def _pb_raw_cid(content: bytes) -> bytes:
|
|
529
|
+
"""CIDv1, codec raw (0x55), sha2-256 (0x12, 32 bytes)."""
|
|
530
|
+
return bytes([0x01, 0x55, 0x12, 0x20]) + hashlib.sha256(content).digest()
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def _pb_file_node(children):
|
|
534
|
+
"""A dag-pb UnixFS File node over `children`, each (cid, tsize, filesize);
|
|
535
|
+
returns (cid, tsize, filesize). Links come before Data in dag-pb; a link
|
|
536
|
+
carries the child's CID, an empty Name and its Tsize, the size of the
|
|
537
|
+
child's subtree; Data is UnixFS Type=File(2), filesize, blocksizes."""
|
|
538
|
+
links = b"".join(
|
|
539
|
+
_pb_field(2, 2, _pb_field(1, 2, cid) + _pb_field(2, 2, b"") + _pb_field(3, 0, _pb_varint(tsize)))
|
|
540
|
+
for cid, tsize, _ in children)
|
|
541
|
+
filesize = sum(size for _, _, size in children)
|
|
542
|
+
data = _pb_field(1, 0, _pb_varint(2)) + _pb_field(3, 0, _pb_varint(filesize))
|
|
543
|
+
data += b"".join(_pb_field(4, 0, _pb_varint(size)) for _, _, size in children)
|
|
544
|
+
block = links + _pb_field(1, 2, data)
|
|
545
|
+
cid = bytes([0x01, 0x70, 0x12, 0x20]) + hashlib.sha256(block).digest()
|
|
546
|
+
return cid, len(block) + sum(tsize for _, tsize, _ in children), filesize
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def unixfs_cid(content: bytes, block_size: int = BLOCK_SIZE) -> str:
|
|
550
|
+
"""The CID the node's Kubo gives `content`: the raw block's CID up to one
|
|
551
|
+
block, else the root of the balanced UnixFS tree over the blocks,
|
|
552
|
+
LINKS_PER_NODE to a node, every node full but the last."""
|
|
553
|
+
if len(content) <= block_size:
|
|
554
|
+
return _cid_b32(_pb_raw_cid(content))
|
|
555
|
+
nodes = [(_pb_raw_cid(content[i:i + block_size]), len(content[i:i + block_size]), len(content[i:i + block_size]))
|
|
556
|
+
for i in range(0, len(content), block_size)]
|
|
557
|
+
while len(nodes) > 1:
|
|
558
|
+
nodes = [_pb_file_node(nodes[i:i + LINKS_PER_NODE]) for i in range(0, len(nodes), LINKS_PER_NODE)]
|
|
559
|
+
return _cid_b32(nodes[0][0])
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def cidv1_raw(content: bytes) -> str:
|
|
563
|
+
"""CIDv1/raw/sha2-256 for `content`: its CID as one raw block, which is
|
|
564
|
+
its CID through the node's Kubo up to BLOCK_SIZE bytes."""
|
|
565
|
+
return _cid_b32(_pb_raw_cid(content))
|
|
490
566
|
|
|
491
567
|
|
|
492
568
|
# AES-GCM nonce length, pinned explicitly. PyCryptodome defaults to 16 bytes
|
|
@@ -601,6 +677,9 @@ class StateRegistry:
|
|
|
601
677
|
contract = self._contract()
|
|
602
678
|
cid, version, _updated = contract.functions.getState(
|
|
603
679
|
self.wallet_address, _key_hash(key)).call()
|
|
680
|
+
own = _task_writes.get(key)
|
|
681
|
+
if own is not None and own[2] > int(version or 0):
|
|
682
|
+
return own
|
|
604
683
|
if not version or not cid:
|
|
605
684
|
return None, default, 0, None
|
|
606
685
|
if not looks_like_cid(cid):
|
|
@@ -747,7 +826,7 @@ class StateRegistry:
|
|
|
747
826
|
|
|
748
827
|
blob = _encrypt(json.dumps(stored, separators=(",", ":")).encode("utf-8"))
|
|
749
828
|
# Compute the CID from OUR bytes -- never accept one from the node.
|
|
750
|
-
cid =
|
|
829
|
+
cid = unixfs_cid(blob)
|
|
751
830
|
|
|
752
831
|
# Hand the blob and the CID to the node for pinning. It verifies the
|
|
753
832
|
# CID matches; it does not get to choose it. Fire-and-forget: the
|
|
@@ -766,6 +845,7 @@ class StateRegistry:
|
|
|
766
845
|
# The ledger records the DATA -- the ACL never leaves in results.
|
|
767
846
|
_ledger_record(key, current_version + 1, cid, new_data)
|
|
768
847
|
_task_nonces[key] = final_nonce
|
|
848
|
+
_task_writes[key] = (new_acl, new_data, current_version + 1, cid)
|
|
769
849
|
return new_acl, new_data
|
|
770
850
|
except Exception as e:
|
|
771
851
|
# A version race means someone else committed first: re-read and
|
|
@@ -805,17 +885,34 @@ class StateRegistry:
|
|
|
805
885
|
delivery path -- but not a trusted one: the content is checked against
|
|
806
886
|
the CID the chain records, so a substituted or corrupted blob is
|
|
807
887
|
rejected rather than decrypted and returned as state.
|
|
888
|
+
|
|
889
|
+
The node stages it as state.<cid>.enc, within seconds of the order's
|
|
890
|
+
start when the object is pinned on it and as it arrives when it has
|
|
891
|
+
to fetch it from the network, so the object is looked for every
|
|
892
|
+
STATE_STAGE_POLL_SECONDS for up to STATE_STAGE_WAIT_SECONDS.
|
|
893
|
+
state.<key>.enc is the last blob a commit of this key published,
|
|
894
|
+
which matches only while no later commit was published; one that
|
|
895
|
+
does not match is passed over.
|
|
808
896
|
"""
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
897
|
+
deadline = time.monotonic() + STATE_STAGE_WAIT_SECONDS
|
|
898
|
+
while True:
|
|
899
|
+
mismatched = []
|
|
900
|
+
for name in (f"state.{cid}.enc", f"state.{key}.enc"):
|
|
901
|
+
ok, data = _swift.get_file_content_bytes(_bucket, name)
|
|
902
|
+
if ok and data:
|
|
903
|
+
if unixfs_cid(data) == cid:
|
|
904
|
+
return data
|
|
905
|
+
mismatched.append(name)
|
|
906
|
+
if time.monotonic() >= deadline:
|
|
907
|
+
break
|
|
908
|
+
time.sleep(STATE_STAGE_POLL_SECONDS)
|
|
909
|
+
if mismatched:
|
|
910
|
+
raise RuntimeError(
|
|
911
|
+
f"State object {' and '.join(mismatched)} does not match the "
|
|
912
|
+
f"committed CID {cid} -- refusing to use it")
|
|
913
|
+
raise RuntimeError(
|
|
914
|
+
f"Could not read state object for '{key}' ({cid}): not staged "
|
|
915
|
+
f"within {STATE_STAGE_WAIT_SECONDS} s")
|
|
819
916
|
|
|
820
917
|
def _send_commit(self, key_hash: bytes, cid: str, expected_version: int,
|
|
821
918
|
owner_addr: str = None, nonce: int = 0):
|
|
@@ -24,6 +24,9 @@ services:
|
|
|
24
24
|
# libraries call os.path.expanduser('~') at startup and abort with
|
|
25
25
|
# "$HOME is not defined" before the enclave emits its public key.
|
|
26
26
|
HOME: /tmp
|
|
27
|
+
# A CAS-provisioned enclave takes its environment from this session,
|
|
28
|
+
# not from the image's ENV; run.js fills in the ESR registry address.
|
|
29
|
+
__ESR_ENVIRONMENT__
|
|
27
30
|
|
|
28
31
|
images:
|
|
29
32
|
- name: application_image
|
package/pynithy/run.js
CHANGED
|
@@ -309,7 +309,8 @@ const main = async () => {
|
|
|
309
309
|
const replacementsSecurelock = {
|
|
310
310
|
PREDECESSOR: PREDECESSOR_HASH_SECURELOCK === 'EMPTY' ? `# predecessor: ${PREDECESSOR_HASH_SECURELOCK}` : `predecessor: ${PREDECESSOR_HASH_SECURELOCK}`,
|
|
311
311
|
MRENCLAVE: process.env.MRENCLAVE_SECURELOCK,
|
|
312
|
-
ENCLAVE_NAME: ENCLAVE_NAME_SECURELOCK
|
|
312
|
+
ENCLAVE_NAME: ENCLAVE_NAME_SECURELOCK,
|
|
313
|
+
ESR_ENVIRONMENT: casConfig.esrSessionEnvironment(process.env)
|
|
313
314
|
};
|
|
314
315
|
if (cas) {
|
|
315
316
|
processYamlTemplate('etny-securelock-test.yaml.tpl', 'etny-securelock-test.yaml', replacementsSecurelock);
|
package/run.js
CHANGED
|
@@ -44,6 +44,7 @@ try { require('dotenv').config(); } catch (e) { /* dotenv optional */ }
|
|
|
44
44
|
const casConfig = require('./cas/config.js');
|
|
45
45
|
const { networkFor, unsafeVariant } = require('./network.js');
|
|
46
46
|
const { PUBLIC_IPFS_HOST } = require('./localKubo.js');
|
|
47
|
+
const { reportRun } = require('./runReport.js');
|
|
47
48
|
|
|
48
49
|
// `--ipfs intake` (the default): the artefacts' CIDs are computed locally and
|
|
49
50
|
// the bytes delivered to the bootnode's payload intake once the DO request is
|
|
@@ -116,20 +117,19 @@ async function main() {
|
|
|
116
117
|
let input = null;
|
|
117
118
|
if (opts.inputFile !== undefined) input = fs.readFileSync(opts.inputFile, 'utf8');
|
|
118
119
|
else if (opts.inputText !== undefined) input = opts.inputText;
|
|
119
|
-
// The
|
|
120
|
-
//
|
|
121
|
-
// ecld-run mirrors ecld-test: input is delivered via the payload call itself.
|
|
120
|
+
// The input is the task's input dataset (runner option inputData): sent
|
|
121
|
+
// sealed for the trustedzone and read in the payload as ___etny_data_set___.
|
|
122
122
|
|
|
123
123
|
// ---- runner (ESM) ----
|
|
124
|
-
let EthernityCloudRunner, ECStatus, ECEvent,
|
|
124
|
+
let EthernityCloudRunner, ECStatus, ECEvent, enums;
|
|
125
125
|
try {
|
|
126
126
|
const runnerMod = await import('@ethernity-cloud/runner');
|
|
127
127
|
EthernityCloudRunner = runnerMod.default || runnerMod.EthernityCloudRunner;
|
|
128
128
|
enums = await import('@ethernity-cloud/runner/enums.js');
|
|
129
|
-
({ ECStatus, ECEvent
|
|
129
|
+
({ ECStatus, ECEvent } = enums);
|
|
130
130
|
} catch (e) {
|
|
131
131
|
console.error('ecld-run: the runner package is not installed or is too old '
|
|
132
|
-
+ '(need @ethernity-cloud/runner >= 0.6.
|
|
132
|
+
+ '(need @ethernity-cloud/runner >= 0.6.11): ' + e.message);
|
|
133
133
|
process.exit(1);
|
|
134
134
|
}
|
|
135
135
|
|
|
@@ -228,10 +228,9 @@ async function main() {
|
|
|
228
228
|
const timeout = new Promise((resolve) => { timer = setTimeout(() => resolve('TIMEOUT'), timeoutMs); });
|
|
229
229
|
|
|
230
230
|
try {
|
|
231
|
-
const runArgs = [resources, securelock, code, opts.node];
|
|
232
|
-
if (trustedzone) runArgs.push(trustedzone);
|
|
233
231
|
// Fire the run; its own promise also settles on completion.
|
|
234
|
-
const runPromise = runner.run(
|
|
232
|
+
const runPromise = runner.run(resources, securelock, code, opts.node, trustedzone || null, { inputData: input })
|
|
233
|
+
.catch((e) => ({ error: e.message }));
|
|
235
234
|
const outcome = await Promise.race([done, timeout, runPromise]);
|
|
236
235
|
clearTimeout(timer);
|
|
237
236
|
|
|
@@ -239,28 +238,7 @@ async function main() {
|
|
|
239
238
|
console.error(`\necld-run: no result after ${opts.timeout}s; the order may still process on-chain.`);
|
|
240
239
|
process.exit(1);
|
|
241
240
|
}
|
|
242
|
-
|
|
243
|
-
console.error(`\nstatus : ERROR`);
|
|
244
|
-
console.error(`error : ${outcome.error}`);
|
|
245
|
-
process.exit(1);
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
// SUCCESS
|
|
249
|
-
const result = await runner.getResult();
|
|
250
|
-
const typed = (runner.getStructuredResult && runner.getStructuredResult()) || null;
|
|
251
|
-
const taskCode = runner.resultTaskCode;
|
|
252
|
-
const taskName = runner.resultTaskCodeName
|
|
253
|
-
|| (ECOrderTaskStatus && taskCode !== undefined ? ECOrderTaskStatus[taskCode] : undefined);
|
|
254
|
-
|
|
255
|
-
if (opts.json) {
|
|
256
|
-
console.log(JSON.stringify({ task_code: taskCode, task_code_name: taskName, value: result, structured: typed }, null, 2));
|
|
257
|
-
} else {
|
|
258
|
-
if (taskName && taskName !== 'SUCCESS') console.log(`task code : ${taskCode} (${taskName})`);
|
|
259
|
-
console.log('result :');
|
|
260
|
-
console.log(result);
|
|
261
|
-
}
|
|
262
|
-
// A non-zero enclave task code is a failed run even though the order landed.
|
|
263
|
-
process.exit(taskCode === undefined || taskCode === 0 ? 0 : 1);
|
|
241
|
+
process.exit(await reportRun(runner, outcome, { json: opts.json }));
|
|
264
242
|
} catch (e) {
|
|
265
243
|
clearTimeout(timer);
|
|
266
244
|
console.error('ecld-run: run failed: ' + (e && e.stack ? e.stack : e));
|
package/runReport.js
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/* ecld-run's report of a finished run.
|
|
4
|
+
*
|
|
5
|
+
* A run that delivered a result prints it; a task the enclave reported with a
|
|
6
|
+
* non-zero task code (an exception in the dApp, a checksum mismatch, ...)
|
|
7
|
+
* prints that code and its result, which is the enclave's error report. The
|
|
8
|
+
* runner signals such a task with its ERROR event, carrying getLastError().
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Print the outcome of a run and return the exit code.
|
|
13
|
+
* outcome: 'SUCCESS', or { error } from the ERROR event or a rejected run().
|
|
14
|
+
*/
|
|
15
|
+
async function reportRun(runner, outcome, { json = false, out = console.log, err = console.error } = {}) {
|
|
16
|
+
// { code, name } of the enclave's task code, or null when no result came.
|
|
17
|
+
const taskCode = runner.getResultTaskCode();
|
|
18
|
+
const error = outcome && outcome.error;
|
|
19
|
+
if (!taskCode && error) {
|
|
20
|
+
err('\nstatus : ERROR');
|
|
21
|
+
err(`error : ${error}`);
|
|
22
|
+
return 1;
|
|
23
|
+
}
|
|
24
|
+
const result = await runner.getResult();
|
|
25
|
+
const typed = (runner.getStructuredResult && runner.getStructuredResult()) || null;
|
|
26
|
+
if (json) {
|
|
27
|
+
out(JSON.stringify({
|
|
28
|
+
task_code: taskCode ? taskCode.code : undefined,
|
|
29
|
+
task_code_name: taskCode ? taskCode.name : undefined,
|
|
30
|
+
value: result,
|
|
31
|
+
structured: typed,
|
|
32
|
+
}, null, 2));
|
|
33
|
+
} else {
|
|
34
|
+
if (taskCode && taskCode.code !== 0) out(`task code : ${taskCode.code} (${taskCode.name})`);
|
|
35
|
+
out('result :');
|
|
36
|
+
out(result);
|
|
37
|
+
}
|
|
38
|
+
if (taskCode && taskCode.code !== 0) {
|
|
39
|
+
err(`error : ${runner.getLastError() || error || taskCode.name}`);
|
|
40
|
+
return 1;
|
|
41
|
+
}
|
|
42
|
+
return error ? 1 : 0;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
module.exports = { reportRun };
|
package/test.js
CHANGED