@temporalio/core-bridge 0.17.2 → 0.18.0

package/Cargo.toml

@@ -10,6 +10,12 @@ exclude = ["index.node", "sdk-core"]
 name = "temporal_sdk_typescript_bridge"
 crate-type = ["cdylib"]
 
+[profile.release]
+opt-level = 3
+debug = false
+lto = true
+incremental = false
+
 [dependencies]
 futures = { version = "0.3", features = ["executor"] }
 log = "0.4"
@@ -19,6 +25,4 @@ prost = "0.9"
 prost-types = "0.9"
 tokio = "1.13"
 once_cell = "1.7.2"
-# TODO: revert back to version *, can't now because of this error: prerelease package needs to be specified explicitly
-temporal-sdk-core = { version = "0.1.0-alpha.1", path = "./sdk-core" }
-temporal-sdk-core-protos = { version = "0.1.0", path = "./sdk-core/sdk-core-protos" }
+temporal-sdk-core = { version = "*", path = "./sdk-core/core" }

package/common.js

@@ -0,0 +1,50 @@
+/**
+ * Shared code for scripts/build.js and index.js
+ *
+ * @module
+ */
+
+const os = require('os');
+const path = require('path');
+const fs = require('fs');
+
+// List of tested compile targets
+const targets = [
+  'x86_64-apple-darwin',
+  'aarch64-apple-darwin',
+  'x86_64-unknown-linux-gnu',
+  'aarch64-unknown-linux-gnu',
+  // TODO: this is not supported on macos
+  'x86_64-pc-windows-msvc',
+  'x86_64-pc-windows-gnu',
+];
+
+const archAlias = { x64: 'x86_64', arm64: 'aarch64' };
+const platformMapping = { darwin: 'apple-darwin', linux: 'unknown-linux-gnu', win32: 'pc-windows-msvc' };
+
+class PrebuildError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'PrebuildError';
+  }
+}
+
+function getPrebuiltPath() {
+  const arch = archAlias[os.arch()];
+  if (arch === undefined) {
+    throw new PrebuildError(`No prebuilt module for arch ${os.arch()}`);
+  }
+  const platform = platformMapping[os.platform()];
+  if (platform === undefined) {
+    throw new PrebuildError(`No prebuilt module for platform ${os.platform()}`);
+  }
+  const binary = path.resolve(__dirname, 'releases', `${arch}-${platform}`, 'index.node');
+  if (fs.existsSync(binary)) {
+    console.log('Found prebuilt bridge module', { binary });
+    return binary;
+  } else {
+    throw new PrebuildError(`No prebuilt module found at ${binary}`);
+  }
+}
+
+module.exports = { targets, archAlias, platformMapping, PrebuildError, getPrebuiltPath };

package/index.d.ts

@@ -169,7 +169,14 @@ export declare type LogsCallback = (err: Error, result: LogEntry[]) => void;
 // TODO: improve type, for some reason Error is not accepted here
 export declare function registerErrors(errors: Record<string, any>): void;
 export declare function newCore(coreOptions: CoreOptions, callback: CoreCallback): void;
+export declare function newReplayCore(telemOptions: TelemetryOptions, callback: CoreCallback): void;
 export declare function newWorker(core: Core, workerOptions: WorkerOptions, callback: WorkerCallback): void;
+export declare function newReplayWorker(
+  core: Core,
+  workerOptions: WorkerOptions,
+  history: ArrayBuffer,
+  callback: WorkerCallback
+): void;
 export declare function workerShutdown(worker: Worker, callback: VoidCallback): void;
 export declare function coreShutdown(core: Core, callback: VoidCallback): void;
 export declare function corePollLogs(core: Core, callback: LogsCallback): void;

package/index.js

@@ -0,0 +1,12 @@
+const { getPrebuiltPath, PrebuildError } = require('./common');
+
+try {
+  const prebuiltPath = getPrebuiltPath();
+  module.exports = require(prebuiltPath);
+} catch (err) {
+  if (err instanceof PrebuildError) {
+    module.exports = require('./default-build/index.node');
+  } else {
+    throw err;
+  }
+}

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@temporalio/core-bridge",
-  "version": "0.17.2",
+  "version": "0.18.0",
   "description": "Temporal.io SDK Core<>Node bridge",
-  "main": "index.node",
+  "main": "index.js",
   "types": "index.d.ts",
   "scripts": {
     "build-rust": "node ./scripts/build.js --force",
     "build": "npm run build-rust",
+    "build-rust-release": "npm run build-rust -- --release",
     "install": "node ./scripts/build.js"
   },
   "keywords": [
@@ -19,7 +20,7 @@
   "license": "MIT",
   "dependencies": {
     "@opentelemetry/api": "^1.0.3",
-    "@temporalio/common": "^0.17.2",
+    "@temporalio/common": "^0.18.0",
     "arg": "^5.0.1",
     "cargo-cp-artifact": "^0.1.4",
     "which": "^2.0.2"
@@ -35,10 +36,12 @@
     "sdk-core",
     "Cargo.toml",
     "Cargo.lock",
+    "index.js",
+    "common.js",
     "index.d.ts"
   ],
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "2232465a4f9b0cade28f0c21c2d7856053678728"
+  "gitHead": "1f8030e0e003fac70969bee9bb816d9520910d02"
 }

package/scripts/build.js

@@ -1,32 +1,19 @@
 const path = require('path');
 const arg = require('arg');
-const os = require('os');
 const fs = require('fs');
 const which = require('which');
 const { spawnSync } = require('child_process');
 const { version } = require('../package.json');
+const { targets, getPrebuiltPath, PrebuildError } = require('../common');
 
 process.chdir(path.resolve(__dirname, '..'));
 
-// List of tested compile targets
-const targets = [
-  'x86_64-apple-darwin',
-  'aarch64-apple-darwin',
-  'x86_64-unknown-linux-gnu',
-  'aarch64-unknown-linux-gnu',
-  // TODO: this is not supported on macos
-  'x86_64-pc-windows-msvc',
-  'x86_64-pc-windows-gnu',
-];
-
-const archAlias = { x64: 'x86_64', arm64: 'aarch64' };
-const platformMapping = { darwin: 'apple-darwin', linux: 'unknown-linux-gnu', win32: 'pc-windows-gnu' };
-
 const args = arg({
   '--help': Boolean,
   '-h': '--help',
   '--version': Boolean,
   '-v': '--version',
+  '--release': Boolean,
   '--target': [String],
   '--force': Boolean,
   '-f': '--force',
@@ -41,7 +28,8 @@ Options:
 -h, --help     Show this help message and exit
 -v, --version  Show program's version number and exit
 -f, --force    Forces a build instead of using a prebuilt binary
--t, --target   Compilation targets, choose any of:
+--release      Build in release mode (or set BUILD_CORE_RELEASE env var)
+--target       Compilation targets, choose any of:
   ${targets.concat('all').join('\n  ')}
 
 Happy compiling!`;
@@ -54,7 +42,6 @@ if (args['--version']) {
   console.log(version);
   process.exit();
 }
-// });
 
 // prepare recompile options
 const targetsArg = args['--target'] || [];
@@ -65,11 +52,12 @@ if (unsupportedTargets.length) {
   process.exit(1);
 }
 const forceBuild = args['--force'];
+const buildRelease = args['--release'] || process.env.BUILD_CORE_RELEASE !== undefined;
 
 function compile(target) {
-  console.log('Compiling bridge', { target });
+  console.log('Compiling bridge', { target, buildRelease });
 
-  const out = target ? `releases/${target}/index.node` : 'index.node';
+  const out = target ? `releases/${target}/index.node` : 'default-build/index.node';
   try {
     fs.unlinkSync(out);
   } catch (err) {
@@ -87,7 +75,7 @@ function compile(target) {
     'cargo',
     'build',
     '--message-format=json-render-diagnostics',
-    '--release',
+    ...(buildRelease ? ['--release'] : []),
     ...(target ? ['--target', target] : []),
   ];
   const cmd = which.sync('cargo-cp-artifact');
@@ -101,35 +89,6 @@ function compile(target) {
   }
 }
 
-class PrebuildError extends Error {
-  constructor(message) {
-    super(message);
-    this.name = 'PrebuildError';
-  }
-}
-
-function usePrebuilt() {
-  const arch = archAlias[os.arch()];
-  if (arch === undefined) {
-    throw new PrebuildError(`No prebuilt module for arch ${os.arch()}`);
-  }
-  const platform = platformMapping[os.platform()];
-  if (arch === undefined) {
-    throw new PrebuildError(`No prebuilt module for platform ${os.platform()}`);
-  }
-  const source = path.resolve(__dirname, '../releases', `${arch}-${platform}`, 'index.node');
-  const target = path.resolve(__dirname, '..', 'index.node');
-  try {
-    fs.copyFileSync(source, target);
-    console.log('Copied prebuilt bridge module', { source, target });
-  } catch (err) {
-    if (err.code === 'ENOENT') {
-      throw new PrebuildError(`No prebuilt module found at ${source}`);
-    }
-    throw err;
-  }
-}
-
 if (requestedTargets.length > 0) {
   // NOTE: no forceBuild
   for (const target of requestedTargets) {
@@ -138,9 +97,10 @@ if (requestedTargets.length > 0) {
 } else {
   if (!forceBuild) {
     try {
-      usePrebuilt();
+      getPrebuiltPath();
     } catch (err) {
       if (err instanceof PrebuildError) {
+        console.warn(err.message);
         compile();
       } else {
         throw err;

package/sdk-core/.buildkite/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.56
+FROM rust:1.58
 
 RUN rustup component add rustfmt && \
 	rustup component add clippy

package/sdk-core/.buildkite/docker/docker-compose.yaml

@@ -9,7 +9,7 @@ services:
   #      - '9042:9042'
 
   temporal:
-    image: temporalio/auto-setup:1.12.3
+    image: temporalio/auto-setup:1.13.0
     ports:
       - "7233:7233"
       - "7234:7234"
@@ -26,7 +26,7 @@ services:
       - cassandra
 
   temporal-web:
-    image: temporalio/web:1.12.0
+    image: temporalio/web:1.13.0
     logging:
       driver: none
     ports:

package/sdk-core/.buildkite/pipeline.yml

@@ -25,6 +25,8 @@ steps:
           config:
             - .buildkite/docker/docker-compose.yaml
             - .buildkite/docker/docker-compose-ci.yaml
+          env:
+            - TEMPORAL_SDK_CORE_BRIDGE_FFI_DISABLE_HEADER_CHANGE=true
   - label: "test"
     agents:
       queue: "default"

package/sdk-core/Cargo.toml

@@ -1,89 +1,2 @@
-[package]
-name = "temporal-sdk-core"
-version = "0.1.0-alpha.1"
-authors = ["Spencer Judge <spencer@temporal.io>", "Vitaly Arbuzov <vitaly@temporal.io>"]
-edition = "2021"
-license-file = "LICENSE.txt"
-description = "Library for building new Temporal SDKs"
-homepage = "https://temporal.io/"
-repository = "https://github.com/temporalio/sdk-core"
-keywords = ["temporal", "workflow"]
-categories = ["development-tools"]
-
-[lib]
-
-[dependencies]
-anyhow = "1.0"
-arc-swap = "1.3"
-async-trait = "0.1"
-backoff = "0.3.0"
-base64 = "0.13"
-crossbeam = "0.8"
-dashmap = "4.0"
-derive_builder = "0.10"
-derive_more = "0.99"
-futures = "0.3"
-futures-retry = "0.6.0"
-http = "0.2"
-hyper = "0.14"
-itertools = "0.10"
-lazy_static = "1.4"
-log = "0.4"
-lru = "0.7"
-once_cell = "1.5"
-opentelemetry = { version = "0.16", features = ["rt-tokio"] }
-opentelemetry-otlp = { version = "0.9.0", features = ["tokio", "metrics"] }
-opentelemetry-prometheus = "0.9.0"
-parking_lot = { version = "0.11", features = ["send_guard"] }
-prometheus = "0.12"
-prost = "0.9"
-prost-types = "0.9"
-rand = "0.8.3"
-ringbuf = "0.2"
-slotmap = "1.0"
-thiserror = "1.0"
-tokio = { version = "1.1", features = ["rt", "rt-multi-thread", "parking_lot", "time", "fs"] }
-tokio-util = { version = "0.6.9" }
-tokio-stream = "0.1"
-tonic = { version = "0.6", features = ["tls", "tls-roots"] }
-tower = "0.4"
-tracing = { version = "0.1", features = ["log-always"] }
-tracing-futures = "0.2"
-tracing-opentelemetry = "0.15"
-tracing-subscriber = { version = "0.2", features = ["parking_lot"] }
-url = "2.2"
-uuid = { version = "0.8.2", features = ["v4"] }
-
-# 1st party local deps
-[dependencies.temporal-sdk-core-protos]
-path = "sdk-core-protos"
-version = "0.1"
-
-[dependencies.rustfsm]
-path = "fsm"
-version = "0.1"
-
-[dev-dependencies]
-assert_matches = "1.4"
-bimap = "0.6.1"
-mockall = "0.10"
-rstest = "0.10"
-test_utils = { path = "test_utils" }
-
-[build-dependencies]
-tonic-build = "0.6"
-
 [workspace]
-members = [".", "fsm", "test_utils", "sdk-core-protos"]
-
-[[test]]
-name = "integ_tests"
-path = "tests/main.rs"
-# Prevents autodiscovery, and hence these getting run with `cargo test`. Run with
-# `cargo test --test integ_tests`
-test = false
-
-[[test]]
-name = "load_tests"
-path = "tests/load_tests.rs"
-test = false
+members = ["bridge-ffi", "core", "client", "core-api", "fsm", "test-utils", "sdk-core-protos", "sdk"]

package/sdk-core/README.md

@@ -6,16 +6,27 @@ Core SDK that can be used as a base for all other Temporal SDKs.
 
 See the [Architecture](ARCHITECTURE.md) doc for some high-level information.
 
-This repo uses a subtree for upstream protobuf files. The path `protos/api_upstream` is a 
-subtree. To update it, use:
-`git subtree pull --prefix protos/api_upstream/ git://github.com/temporalio/api.git master --squash`
-
 ## Dependencies
 * Protobuf compiler
 
 # Development
 
-All of the following commands are enforced for each pull request.
+This repo is composed of multiple crates:
+* temporal-sdk-core-protos `./sdk-core-protos` - Holds the generated proto code and extensions
+* temporal-client `./client` - Defines client(s) for interacting with the Temporal gRPC service
+* temporal-sdk-core-api `./core-api` - Defines the API surface exposed by Core
+* temporal-sdk-core `./core` - The Core implementation
+* temporal-sdk `./sdk` - A (currently prototype) Rust SDK built on top of Core. Used for testing.
+* temporal-sdk-core-bridge-ffi `./bridge-ffi` - C API wrapper for Core
+* rustfsm `./fsm` - Implements a procedural macro used by core for defining state machines
+    (contains subcrates). It is temporal agnostic.
+
+Visualized (dev dependencies are in blue):
+
+![Crate dependency graph](./etc/deps.svg)
+
+
+All the following commands are enforced for each pull request:
 
 ## Building and testing
 
@@ -52,10 +63,23 @@ To run integ tests with OTel collection on, you can use `integ-with-otel.sh`. Yo
 sure you are running the collector via docker, which can be done like so:
 `docker-compose -f .buildkite/docker/docker-compose.yaml -f .buildkite/docker/docker-compose-telem.yaml up`
 
-
 If you are working on a language SDK, you are expected to initialize tracing early in your `main`
 equivalent.
 
+## Proto files
+
+This repo uses a subtree for upstream protobuf files. The path `protos/api_upstream` is a
+subtree. To update it, use:
+`git subtree pull --prefix protos/api_upstream/ git://github.com/temporalio/api.git master --squash`
+
+## Fetching Histories
+Tests which would like to replay stored histories rely on that history being made available in
+binary format. You can fetch histories in that format like so (from a local docker server):
+
+`cargo run --bin histfetch {workflow_id} [{run_id}]`
+
+You can change the `TEMPORAL_SERVICE_ADDRESS` env var to fetch from a different address.
+
 ## Style Guidelines
 
 ### Error handling

package/sdk-core/bridge-ffi/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "temporal-sdk-core-bridge-ffi"
+version = "0.1.0"
+edition = "2021"
+license-file = "LICENSE.txt"
+description = "FFI Bridge for Core"
+homepage = "https://temporal.io/"
+repository = "https://github.com/temporalio/sdk-core"
+
+[lib]
+crate-type = ["staticlib"]
+
+[dependencies]
+lazy_static = "1.4"
+libc = "0.2"
+log = "0.4"
+prost = "0.9"
+temporal-sdk-core = { version = "0.1", path = "../core" }
+temporal-sdk-core-api = { version = "0.1", path = "../core-api" }
+temporal-sdk-core-protos = { version = "0.1", path = "../sdk-core-protos" }
+tokio = "1"
+
+[build-dependencies]
+cbindgen = "0.20.0"

package/sdk-core/bridge-ffi/LICENSE.txt

@@ -0,0 +1,23 @@
+Temporal Core SDK
+
+The MIT License
+
+Copyright (c) 2022 Temporal Technologies, Inc. All Rights Reserved
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/sdk-core/bridge-ffi/build.rs

@@ -0,0 +1,25 @@
+extern crate cbindgen;
+
+use std::env;
+
+fn main() {
+    let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
+
+    let changed = cbindgen::Builder::new()
+        .with_crate(crate_dir)
+        .with_pragma_once(true)
+        .with_language(cbindgen::Language::C)
+        .generate()
+        .expect("Unable to generate bindings")
+        .write_to_file("include/sdk-core-bridge.h");
+
+    // If this changed and an env var disallows change, error
+    if let Ok(env_val) = env::var("TEMPORAL_SDK_CORE_BRIDGE_FFI_DISABLE_HEADER_CHANGE") {
+        if changed && env_val == "true" {
+            println!(
+                "cargo:warning=bridge-ffi's header file changed unexpectedly from what's on disk"
+            );
+            std::process::exit(1);
+        }
+    }
+}