@ethernity-cloud/sdk-js 1.7.17 → 1.7.20

package/nodenithy/build/securelock/Dockerfile.tmpl

@@ -63,8 +63,9 @@ RUN openssl genrsa -3 -out /enclave-key.pem 3072
 # scone-signer below (SCONE embeds defaults for anything not signed explicitly;
 # any drift makes SCONE recompute + debug re-sign the enclave at load, which CAS
 # rejects on mainnet). Values mirror the publish/run docker-compose securelock
-# service (SCONE_HEAP=1024M, SCONE_STACK=4M, dlopen=1, extensions=libbinary-fs).
-ENV SCONE_HEAP=1024M
+# service (SCONE_HEAP as build.mjs's MEMORY_TO_ALLOCATE, SCONE_STACK=4M,
+# dlopen=1, extensions=libbinary-fs).
+ENV SCONE_HEAP=__MEMORY_TO_ALLOCATE__
 ENV SCONE_STACK=4M
 ENV SCONE_ALLOW_DLOPEN=1
 ENV SCONE_EXTENSIONS_PATH=/lib/libbinary-fs.so

package/nodenithy/build.mjs

@@ -265,7 +265,12 @@ let dockerfileSecureContent = dockerfileSecureTemplate.replace(/__ENCLAVE_NAME_S
 
 // Amount of enclave heap to allocate (SCONE_HEAP). Kept in sync with the
 // run/docker-compose securelock service; overridable via ECLD_MEMORY_TO_ALLOCATE.
+// 1024M: with 256M or 512M the node runtime exhausts the protected heap
+// reserving its V8 code range at start, before the securelock runs.
 const MEMORY_TO_ALLOCATE = (process.env.ECLD_MEMORY_TO_ALLOCATE || '1024M').trim();
+// Read back by ecld-publish (run.js), which renders it into the composes'
+// securelock service.
+writeEnv('MEMORY_TO_ALLOCATE', MEMORY_TO_ALLOCATE);
 
 // CRITICAL (mainnet DCAP): sign /usr/local/bin/node -- the binary the enclave
 // actually EXECUTES (ENTRYPOINT + the run/publish compose command both run
@@ -297,6 +302,7 @@ const signedMrenclaveStep =
 // securelock is signed debug with the same explicit params.
 const cas = casConfig.casProvisioned(templateName, isMainnet);
 dockerfileSecureContent = dockerfileSecureContent
+  .replace(/__MEMORY_TO_ALLOCATE__/g, MEMORY_TO_ALLOCATE)
   .replace('__SCONE_SIGN__', `RUN scone-signer sign ${signFlags}${cas ? ' --production' : ''} /usr/local/bin/node`)
   .replace('__SIGNED_MRENCLAVE__', signedMrenclaveStep);
 

package/nodenithy/run/docker-compose-final.yml.tmpl

@@ -36,9 +36,10 @@ services:
       - SCONE_CAS_ADDR=scone-cas.cf  # __CAS_ONLY__
       - SCONE_LAS_ADDR=las  # __CAS_ONLY__
       - SCONE_CONFIG_ID=__ENCLAVE_NAME_SECURELOCK__/application  # __CAS_ONLY__
-      - SCONE_HEAP=1024M
-      # Must match the --stack the securelock binary was signed with (build.mjs),
-      # otherwise SCONE recomputes MRENCLAVE at load and re-signs as debug.
+      # Must match the --heap and --stack the securelock binary was signed with
+      # (build.mjs), otherwise SCONE recomputes MRENCLAVE at load and re-signs
+      # as debug. The heap is the build's MEMORY_TO_ALLOCATE (.env).
+      - SCONE_HEAP=__MEMORY_TO_ALLOCATE__
       - SCONE_STACK=4M
       - SCONE_LOG=DEBUG
       - SCONE_ALLOW_DLOPEN=1
@@ -55,8 +56,8 @@ services:
     container_name: etny-trustedzone
     privileged: true
     image: localhost:5000/etny-trustedzone
-    entrypoint: ""
-    command: [ "/usr/local/bin/node", "/etny-trustedzone/trustedzone.js" ]
+    # No command: the image's ENTRYPOINT runs node with the flags the
+    # trustedzone was built and measured with (etny-nodenithy Dockerfile.tmpl).
     environment:
       - SCONE_CAS_ADDR=scone-cas.cf  # __CAS_ONLY__
       - SCONE_LAS_ADDR=las  # __CAS_ONLY__

package/nodenithy/run/docker-compose.yml.tmpl

@@ -39,9 +39,10 @@ services:
       - SCONE_CAS_ADDR=scone-cas.cf  # __CAS_ONLY__
       - SCONE_LAS_ADDR=las  # __CAS_ONLY__
       - SCONE_CONFIG_ID=__ENCLAVE_NAME_SECURELOCK__/application  # __CAS_ONLY__
-      - SCONE_HEAP=1024M
-      # Must match the --stack the securelock binary was signed with (build.mjs),
-      # otherwise SCONE recomputes MRENCLAVE at load and re-signs as debug.
+      # Must match the --heap and --stack the securelock binary was signed with
+      # (build.mjs), otherwise SCONE recomputes MRENCLAVE at load and re-signs
+      # as debug. The heap is the build's MEMORY_TO_ALLOCATE (.env).
+      - SCONE_HEAP=__MEMORY_TO_ALLOCATE__
       - SCONE_STACK=4M
       - SCONE_LOG=DEBUG
       - SCONE_ALLOW_DLOPEN=1
@@ -54,14 +55,14 @@ services:
     container_name: etny-trustedzone
     privileged: true
     image: localhost:5000/etny-trustedzone
-    entrypoint: ""
-    command: [ "/usr/local/bin/node", "/etny-trustedzone/trustedzone.js" ]
+    # No command: the image's ENTRYPOINT runs node with the flags the
+    # trustedzone was built and measured with (etny-nodenithy Dockerfile.tmpl).
     environment:
       - SCONE_CAS_ADDR=scone-cas.cf  # __CAS_ONLY__
       - SCONE_LAS_ADDR=las  # __CAS_ONLY__
       - SCONE_CONFIG_ID=__ENCLAVE_NAME_TRUSTEDZONE__/application  # __CAS_ONLY__
-      # The heap the trustedzone was measured and registered with: 1024M for
-      # every testnet trustedzone (etny-nodenithy v3/run/docker-compose.yml).
+      # The heap the trustedzone was signed with: the SCONE_HEAP of the
+      # bundled trustedzone image (run.js reads it from the image).
       - SCONE_HEAP=__TRUSTEDZONE_HEAP__
       - SCONE_LOG=DEBUG
       - SCONE_ALLOW_DLOPEN=1

package/nodenithy/run/public_key_service.js

@@ -5,7 +5,8 @@
 // registries) is queued by the service's registry watcher from the
 // ImageRegistered event, with this publish's IPFS peer; the hash is posted to
 // /api/addHash only when that row has not appeared within
-// CHAIN_ROW_GRACE_SECONDS. Other images are posted at once.
+// CHAIN_ROW_GRACE_SECONDS, or when the service holds a failed extraction of
+// the image. Other images are posted at once.
 const axios = require('axios');
 const fs = require('fs');
 const { Command } = require('commander');
@@ -14,6 +15,14 @@ const BASE_URL = "https://publickey.ethernity.cloud";
 // The watcher polls the registries every 30 s.
 const CHAIN_ROW_GRACE_SECONDS = 180;
 const POLL_SECONDS = 10;
+// A failed extraction is recorded against the hash with publicKey -1 and a
+// reason, and a new submission of the hash runs it again. The service answers
+// the old record for a moment after a submission, so a failed answer is taken
+// as the outcome only once this long has passed since this publish submitted.
+const RESUBMIT_GRACE_SECONDS = 60;
+
+// The service's record of an extraction that failed.
+const failed = (answer) => answer !== null && answer.publicKey === -1;
 
 async function submitIpfsHash(hash, enclaveName, protocolVersion, network, templateVersion, dockerComposerHash) {
     const url = `${BASE_URL}/api/addHash`;
@@ -48,9 +57,10 @@ async function checkIpfsHashStatus(hash) {
     }
 }
 
-// Queues the image as described above and returns the service's first
-// answer for it. `submit`, `check`, `sleep` and `now` are the functions
-// above and the clock, replaceable for tests.
+// Queues the image as described above and returns the service's first answer
+// for it (`answer`, null right after a submission) and whether this publish
+// posted the hash (`submitted`). `submit`, `check`, `sleep` and `now` are the
+// functions above and the clock, replaceable for tests.
 async function queueImage({ hash, chainRegistered, submitArgs, submit = submitIpfsHash, check = checkIpfsHashStatus,
     sleep = (s) => new Promise((resolve) => setTimeout(resolve, s * 1000)), now = () => Date.now() / 1000,
     log = console.log, graceSeconds = CHAIN_ROW_GRACE_SECONDS, pollSeconds = POLL_SECONDS }) {
@@ -58,7 +68,11 @@ async function queueImage({ hash, chainRegistered, submitArgs, submit = submitIp
         const deadline = now() + graceSeconds;
         while (true) {
             const answer = await check(hash);
-            if (answer !== null) return answer;
+            if (failed(answer)) {
+                log(`The service's previous extraction of this image failed (${answer.reason || "no reason reported"}); submitting the hash again`);
+                break;
+            }
+            if (answer !== null) return { answer, submitted: false };
             if (now() >= deadline) {
                 log(`The certificate service has not picked the registration up from the chain in ${graceSeconds} s; submitting the hash`);
                 break;
@@ -69,7 +83,7 @@ async function queueImage({ hash, chainRegistered, submitArgs, submit = submitIp
     }
     const submitResponse = await submit(hash, ...submitArgs);
     log("Submit IPFS Hash Response:", submitResponse);
-    return null;
+    return { answer: null, submitted: true };
 }
 
 async function main() {
@@ -98,10 +112,12 @@ async function main() {
     console.log("Template Version:", templateVersion);
     console.log("Docker Composer Hash:", dockerComposerHash);
 
-    let checkResponse = await queueImage({
+    const queued = await queueImage({
         hash: hhash, chainRegistered: Boolean(options.chainRegistered),
         submitArgs: [enclaveName, protocolVersion, network, templateVersion, dockerComposerHash],
     });
+    let checkResponse = queued.answer;
+    const submittedAt = queued.submitted ? Date.now() / 1000 : null;
 
     // The service answers publicKey 0 while the image is queued or running
     // (with status, percent and queuePosition), -1 when the extraction failed
@@ -119,10 +135,15 @@ async function main() {
                 } else {
                     console.log(`Public key not available yet. Queue position: ${checkResponse.queuePosition ?? 'Unknown'}`);
                 }
-            } else if (checkResponse.publicKey === -1) {
-                console.log(`Public key extraction failed: ${checkResponse.reason || "reason not reported by the service"}`);
-                console.log("Fix the cause (keep the image pinned, build with the current SDK) and publish again: a new submission of the same hash runs the extraction again.");
-                process.exit(1);
+            } else if (failed(checkResponse)) {
+                if (submittedAt !== null && Date.now() / 1000 - submittedAt < RESUBMIT_GRACE_SECONDS) {
+                    // The record of the run before this publish's submission.
+                    console.log("The service still answers its previous record; waiting for the new run");
+                } else {
+                    console.log(`Public key extraction failed: ${checkResponse.reason || "reason not reported by the service"}`);
+                    console.log("Fix the cause (keep the image pinned, build with the current SDK) and publish again: a new submission of the same hash runs the extraction again.");
+                    process.exit(1);
+                }
             } else {
                 console.log("Public Key:", checkResponse.publicKey);
                 // Save public key to file
@@ -152,4 +173,4 @@ async function main() {
 
 if (require.main === module) main();
 
-module.exports = { queueImage, CHAIN_ROW_GRACE_SECONDS, POLL_SECONDS };
+module.exports = { queueImage, CHAIN_ROW_GRACE_SECONDS, POLL_SECONDS, RESUBMIT_GRACE_SECONDS };

package/nodenithy/run.js

@@ -113,12 +113,28 @@ const extractSignedMrenclave = (service) => {
     }
 };
 
+// The heap the trustedzone was signed with: the SCONE_HEAP of the image the
+// build bundled (etny-nodenithy's Dockerfile sets it to the --heap it signs
+// with). The compose must name the same value, or SCONE recomputes the
+// measurement at load and the CAS refuses the enclave.
+const trustedZoneHeapOfImage = (image) => {
+    const env = execSync(`docker image inspect ${image} --format "{{range .Config.Env}}{{println .}}{{end}}"`,
+        { stdio: ['pipe', 'pipe', 'pipe'] }).toString();
+    const m = env.match(/^SCONE_HEAP=(\S+)$/m);
+    if (!m) throw new Error(`${image} carries no SCONE_HEAP in its environment`);
+    return m[1];
+};
+
 const main = async () => {
     process.env.NODE_NO_WARNINGS = 1
-    // Every testnet trustedzone was measured and registered with a 1024M heap
-    // (etny-nodenithy v3/run/docker-compose.yml). The mainnet session named
-    // below comes from the features/v3 branches, which ran it with 256M.
-    const trustedZoneHeap = isMainnet ? '256M' : '1024M';
+    const trustedZoneHeap = trustedZoneHeapOfImage('localhost:5000/etny-trustedzone');
+    console.log(`Trustedzone heap: ${trustedZoneHeap}`);
+    // The securelock heap ecld-build signed it with (part of MRENCLAVE).
+    const memory = (process.env.MEMORY_TO_ALLOCATE || '').trim();
+    if (!memory) {
+        console.error("Error: MEMORY_TO_ALLOCATE is not set in .env; run ecld-build first.");
+        process.exit(1);
+    }
 
     // Where the image goes. An IPFS API of the application's own
     // (IPFS_ENDPOINT, with IPFS_TOKEN as its Authorization header when set)
@@ -208,7 +224,8 @@ const main = async () => {
             return;
         }
         const content = casConfig.renderCompose(fs.readFileSync(template, 'utf8'), cas, unsafe)
-            .replace(/__TRUSTEDZONE_HEAP__/g, trustedZoneHeap);
+            .replace(/__TRUSTEDZONE_HEAP__/g, trustedZoneHeap)
+            .replace(/__MEMORY_TO_ALLOCATE__/g, memory);
         fs.writeFileSync(file, content, 'utf8');
     });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ethernity-cloud/sdk-js",
-  "version": "1.7.17",
+  "version": "1.7.20",
   "description": "A package to run ethernity cloud build scripts",
   "scripts": {
     "test": "node --test \"test/**/*.test.js\"",

package/pynithy/build.mjs

@@ -156,7 +156,7 @@ const casProvisioned = casConfig.casProvisioned(templateName, isMainnet);
 
 // Enclave heap (SCONE_HEAP). Part of the measurement: publish runs the
 // securelock with the value recorded here.
-const MEMORY_TO_ALLOCATE = (process.env.ECLD_MEMORY_TO_ALLOCATE || process.env.MEMORY_TO_ALLOCATE || '1024M').trim();
+const MEMORY_TO_ALLOCATE = (process.env.ECLD_MEMORY_TO_ALLOCATE || process.env.MEMORY_TO_ALLOCATE || '256M').trim();
 writeEnv('MEMORY_TO_ALLOCATE', MEMORY_TO_ALLOCATE);
 
 runCommand('docker pull registry:2');

package/pynithy/run/public_key_service.js

@@ -5,7 +5,8 @@
 // registries) is queued by the service's registry watcher from the
 // ImageRegistered event, with this publish's IPFS peer; the hash is posted to
 // /api/addHash only when that row has not appeared within
-// CHAIN_ROW_GRACE_SECONDS. Other images are posted at once.
+// CHAIN_ROW_GRACE_SECONDS, or when the service holds a failed extraction of
+// the image. Other images are posted at once.
 const axios = require('axios');
 const fs = require('fs');
 const { Command } = require('commander');
@@ -14,6 +15,14 @@ const BASE_URL = "https://publickey.ethernity.cloud";
 // The watcher polls the registries every 30 s.
 const CHAIN_ROW_GRACE_SECONDS = 180;
 const POLL_SECONDS = 10;
+// A failed extraction is recorded against the hash with publicKey -1 and a
+// reason, and a new submission of the hash runs it again. The service answers
+// the old record for a moment after a submission, so a failed answer is taken
+// as the outcome only once this long has passed since this publish submitted.
+const RESUBMIT_GRACE_SECONDS = 60;
+
+// The service's record of an extraction that failed.
+const failed = (answer) => answer !== null && answer.publicKey === -1;
 
 async function submitIpfsHash(hash, enclaveName, protocolVersion, network, templateVersion, dockerComposerHash) {
     const url = `${BASE_URL}/api/addHash`;
@@ -48,9 +57,10 @@ async function checkIpfsHashStatus(hash) {
     }
 }
 
-// Queues the image as described above and returns the service's first
-// answer for it. `submit`, `check`, `sleep` and `now` are the functions
-// above and the clock, replaceable for tests.
+// Queues the image as described above and returns the service's first answer
+// for it (`answer`, null right after a submission) and whether this publish
+// posted the hash (`submitted`). `submit`, `check`, `sleep` and `now` are the
+// functions above and the clock, replaceable for tests.
 async function queueImage({ hash, chainRegistered, submitArgs, submit = submitIpfsHash, check = checkIpfsHashStatus,
     sleep = (s) => new Promise((resolve) => setTimeout(resolve, s * 1000)), now = () => Date.now() / 1000,
     log = console.log, graceSeconds = CHAIN_ROW_GRACE_SECONDS, pollSeconds = POLL_SECONDS }) {
@@ -58,7 +68,11 @@ async function queueImage({ hash, chainRegistered, submitArgs, submit = submitIp
         const deadline = now() + graceSeconds;
         while (true) {
             const answer = await check(hash);
-            if (answer !== null) return answer;
+            if (failed(answer)) {
+                log(`The service's previous extraction of this image failed (${answer.reason || "no reason reported"}); submitting the hash again`);
+                break;
+            }
+            if (answer !== null) return { answer, submitted: false };
             if (now() >= deadline) {
                 log(`The certificate service has not picked the registration up from the chain in ${graceSeconds} s; submitting the hash`);
                 break;
@@ -69,7 +83,7 @@ async function queueImage({ hash, chainRegistered, submitArgs, submit = submitIp
     }
     const submitResponse = await submit(hash, ...submitArgs);
     log("Submit IPFS Hash Response:", submitResponse);
-    return null;
+    return { answer: null, submitted: true };
 }
 
 async function main() {
@@ -98,10 +112,12 @@ async function main() {
     console.log("Template Version:", templateVersion);
     console.log("Docker Composer Hash:", dockerComposerHash);
 
-    let checkResponse = await queueImage({
+    const queued = await queueImage({
         hash: hhash, chainRegistered: Boolean(options.chainRegistered),
         submitArgs: [enclaveName, protocolVersion, network, templateVersion, dockerComposerHash],
     });
+    let checkResponse = queued.answer;
+    const submittedAt = queued.submitted ? Date.now() / 1000 : null;
 
     // The service answers publicKey 0 while the image is queued or running
     // (with status, percent and queuePosition), -1 when the extraction failed
@@ -119,10 +135,15 @@ async function main() {
                 } else {
                     console.log(`Public key not available yet. Queue position: ${checkResponse.queuePosition ?? 'Unknown'}`);
                 }
-            } else if (checkResponse.publicKey === -1) {
-                console.log(`Public key extraction failed: ${checkResponse.reason || "reason not reported by the service"}`);
-                console.log("Fix the cause (keep the image pinned, build with the current SDK) and publish again: a new submission of the same hash runs the extraction again.");
-                process.exit(1);
+            } else if (failed(checkResponse)) {
+                if (submittedAt !== null && Date.now() / 1000 - submittedAt < RESUBMIT_GRACE_SECONDS) {
+                    // The record of the run before this publish's submission.
+                    console.log("The service still answers its previous record; waiting for the new run");
+                } else {
+                    console.log(`Public key extraction failed: ${checkResponse.reason || "reason not reported by the service"}`);
+                    console.log("Fix the cause (keep the image pinned, build with the current SDK) and publish again: a new submission of the same hash runs the extraction again.");
+                    process.exit(1);
+                }
             } else {
                 console.log("Public Key:", checkResponse.publicKey);
                 // Save public key to file
@@ -152,4 +173,4 @@ async function main() {
 
 if (require.main === module) main();
 
-module.exports = { queueImage, CHAIN_ROW_GRACE_SECONDS, POLL_SECONDS };
+module.exports = { queueImage, CHAIN_ROW_GRACE_SECONDS, POLL_SECONDS, RESUBMIT_GRACE_SECONDS };