@temporalio/core-bridge 0.13.0 → 0.16.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +203 -78
- package/Cargo.toml +3 -3
- package/index.d.ts +195 -0
- package/index.node +0 -0
- package/package.json +10 -6
- package/releases/aarch64-apple-darwin/index.node +0 -0
- package/releases/{x86_64-pc-windows-gnu → aarch64-unknown-linux-gnu}/index.node +0 -0
- package/releases/x86_64-apple-darwin/index.node +0 -0
- package/releases/x86_64-pc-windows-msvc/index.node +0 -0
- package/releases/x86_64-unknown-linux-gnu/index.node +0 -0
- package/scripts/build.js +77 -34
- package/sdk-core/.buildkite/docker/Dockerfile +1 -1
- package/sdk-core/CODEOWNERS +1 -1
- package/sdk-core/Cargo.toml +6 -5
- package/sdk-core/fsm/Cargo.toml +1 -1
- package/sdk-core/fsm/rustfsm_procmacro/Cargo.toml +2 -2
- package/sdk-core/fsm/rustfsm_procmacro/src/lib.rs +8 -9
- package/sdk-core/fsm/rustfsm_procmacro/tests/trybuild/no_handle_conversions_require_into_fail.stderr +13 -7
- package/sdk-core/fsm/rustfsm_trait/Cargo.toml +2 -2
- package/sdk-core/fsm/rustfsm_trait/src/lib.rs +1 -1
- package/sdk-core/protos/local/activity_result.proto +10 -1
- package/sdk-core/protos/local/workflow_activation.proto +6 -3
- package/sdk-core/sdk-core-protos/Cargo.toml +4 -4
- package/sdk-core/sdk-core-protos/src/lib.rs +44 -49
- package/sdk-core/src/core_tests/activity_tasks.rs +5 -5
- package/sdk-core/src/core_tests/child_workflows.rs +55 -29
- package/sdk-core/src/core_tests/determinism.rs +19 -9
- package/sdk-core/src/core_tests/mod.rs +3 -3
- package/sdk-core/src/core_tests/retry.rs +96 -2
- package/sdk-core/src/core_tests/workers.rs +1 -1
- package/sdk-core/src/core_tests/workflow_tasks.rs +278 -4
- package/sdk-core/src/errors.rs +27 -44
- package/sdk-core/src/lib.rs +13 -3
- package/sdk-core/src/machines/activity_state_machine.rs +44 -5
- package/sdk-core/src/machines/child_workflow_state_machine.rs +31 -11
- package/sdk-core/src/machines/complete_workflow_state_machine.rs +1 -1
- package/sdk-core/src/machines/continue_as_new_workflow_state_machine.rs +1 -1
- package/sdk-core/src/machines/mod.rs +18 -23
- package/sdk-core/src/machines/patch_state_machine.rs +8 -8
- package/sdk-core/src/machines/signal_external_state_machine.rs +22 -1
- package/sdk-core/src/machines/timer_state_machine.rs +21 -3
- package/sdk-core/src/machines/transition_coverage.rs +3 -3
- package/sdk-core/src/machines/workflow_machines.rs +11 -11
- package/sdk-core/src/pending_activations.rs +27 -22
- package/sdk-core/src/pollers/gateway.rs +28 -7
- package/sdk-core/src/pollers/poll_buffer.rs +6 -5
- package/sdk-core/src/pollers/retry.rs +193 -136
- package/sdk-core/src/prototype_rust_sdk/workflow_context.rs +61 -46
- package/sdk-core/src/prototype_rust_sdk/workflow_future.rs +13 -12
- package/sdk-core/src/prototype_rust_sdk.rs +17 -23
- package/sdk-core/src/telemetry/metrics.rs +2 -4
- package/sdk-core/src/telemetry/mod.rs +6 -7
- package/sdk-core/src/test_help/canned_histories.rs +17 -93
- package/sdk-core/src/test_help/history_builder.rs +51 -2
- package/sdk-core/src/test_help/history_info.rs +2 -2
- package/sdk-core/src/test_help/mod.rs +21 -34
- package/sdk-core/src/worker/activities/activity_heartbeat_manager.rs +246 -138
- package/sdk-core/src/worker/activities.rs +47 -45
- package/sdk-core/src/worker/config.rs +11 -0
- package/sdk-core/src/worker/dispatcher.rs +5 -5
- package/sdk-core/src/worker/mod.rs +86 -56
- package/sdk-core/src/workflow/driven_workflow.rs +3 -3
- package/sdk-core/src/workflow/history_update.rs +1 -1
- package/sdk-core/src/workflow/mod.rs +2 -1
- package/sdk-core/src/workflow/workflow_tasks/cache_manager.rs +13 -17
- package/sdk-core/src/workflow/workflow_tasks/concurrency_manager.rs +10 -18
- package/sdk-core/src/workflow/workflow_tasks/mod.rs +72 -57
- package/sdk-core/test_utils/Cargo.toml +1 -1
- package/sdk-core/test_utils/src/lib.rs +2 -2
- package/sdk-core/tests/integ_tests/workflow_tests/activities.rs +131 -2
- package/sdk-core/tests/integ_tests/workflow_tests/child_workflows.rs +2 -2
- package/sdk-core/tests/integ_tests/workflow_tests/determinism.rs +49 -0
- package/sdk-core/tests/integ_tests/workflow_tests/signals.rs +2 -2
- package/sdk-core/tests/integ_tests/workflow_tests.rs +74 -47
- package/src/conversions.rs +17 -0
- package/src/errors.rs +0 -7
- package/src/lib.rs +0 -20
|
@@ -6,7 +6,7 @@ use crate::{
|
|
|
6
6
|
build_fake_core, build_mock_pollers, build_multihist_mock_sg, canned_histories,
|
|
7
7
|
gen_assert_and_fail, gen_assert_and_reply, hist_to_poll_resp, mock_core, poll_and_reply,
|
|
8
8
|
poll_and_reply_clears_outstanding_evicts, single_hist_mock_sg, FakeWfResponses,
|
|
9
|
-
MockPollCfg, MocksHolder, ResponseType, TestHistoryBuilder, TEST_Q,
|
|
9
|
+
MockPollCfg, MocksHolder, ResponseType, TestHistoryBuilder, NO_MORE_WORK_ERROR_MSG, TEST_Q,
|
|
10
10
|
},
|
|
11
11
|
workflow::WorkflowCachingPolicy::{self, AfterEveryReply, NonSticky},
|
|
12
12
|
Core, CoreSDK, WfActivationCompletion,
|
|
@@ -30,7 +30,7 @@ use temporal_sdk_core_protos::{
|
|
|
30
30
|
},
|
|
31
31
|
},
|
|
32
32
|
temporal::api::{
|
|
33
|
-
enums::v1::EventType,
|
|
33
|
+
enums::v1::{EventType, WorkflowTaskFailedCause},
|
|
34
34
|
failure::v1::Failure,
|
|
35
35
|
history::v1::{history_event, TimerFiredEventAttributes},
|
|
36
36
|
workflowservice::v1::RespondWorkflowTaskCompletedResponse,
|
|
@@ -642,9 +642,9 @@ async fn workflow_update_random_seed_on_workflow_reset() {
|
|
|
642
642
|
UpdateRandomSeed{randomness_seed})),
|
|
643
643
|
}] => {
|
|
644
644
|
assert_ne!(randomness_seed_from_start.load(Ordering::SeqCst),
|
|
645
|
-
*randomness_seed)
|
|
645
|
+
*randomness_seed);
|
|
646
646
|
}
|
|
647
|
-
)
|
|
647
|
+
);
|
|
648
648
|
},
|
|
649
649
|
vec![CompleteWorkflowExecution { result: None }.into()],
|
|
650
650
|
),
|
|
@@ -1391,3 +1391,277 @@ async fn buffering_tasks_doesnt_count_toward_outstanding_max() {
|
|
|
1391
1391
|
PollWfError::TonicError(_)
|
|
1392
1392
|
);
|
|
1393
1393
|
}
|
|
1394
|
+
|
|
1395
|
+
#[tokio::test]
|
|
1396
|
+
async fn fail_wft_then_recover() {
|
|
1397
|
+
let t = canned_histories::long_sequential_timers(1);
|
|
1398
|
+
let mut mh = MockPollCfg::from_resp_batches(
|
|
1399
|
+
"fake_wf_id",
|
|
1400
|
+
t,
|
|
1401
|
+
// We need to deliver all of history twice because of eviction
|
|
1402
|
+
[ResponseType::AllHistory, ResponseType::AllHistory],
|
|
1403
|
+
MockServerGatewayApis::new(),
|
|
1404
|
+
);
|
|
1405
|
+
mh.num_expected_fails = Some(1);
|
|
1406
|
+
mh.expect_fail_wft_matcher =
|
|
1407
|
+
Box::new(|_, cause, _| matches!(cause, WorkflowTaskFailedCause::NonDeterministicError));
|
|
1408
|
+
let mut mock = build_mock_pollers(mh);
|
|
1409
|
+
mock.worker_cfg(TEST_Q, |wc| {
|
|
1410
|
+
wc.max_cached_workflows = 2;
|
|
1411
|
+
});
|
|
1412
|
+
let core = mock_core(mock);
|
|
1413
|
+
|
|
1414
|
+
let act = core.poll_workflow_activation(TEST_Q).await.unwrap();
|
|
1415
|
+
// Start an activity instead of a timer, triggering nondeterminism error
|
|
1416
|
+
core.complete_workflow_activation(WfActivationCompletion::from_cmds(
|
|
1417
|
+
TEST_Q,
|
|
1418
|
+
act.run_id.clone(),
|
|
1419
|
+
vec![ScheduleActivity {
|
|
1420
|
+
activity_id: "fake_activity".to_string(),
|
|
1421
|
+
..Default::default()
|
|
1422
|
+
}
|
|
1423
|
+
.into()],
|
|
1424
|
+
))
|
|
1425
|
+
.await
|
|
1426
|
+
.unwrap();
|
|
1427
|
+
// We must handle an eviction now
|
|
1428
|
+
let evict_act = core.poll_workflow_activation(TEST_Q).await.unwrap();
|
|
1429
|
+
assert_eq!(evict_act.run_id, act.run_id);
|
|
1430
|
+
assert_matches!(
|
|
1431
|
+
evict_act.jobs.as_slice(),
|
|
1432
|
+
[WfActivationJob {
|
|
1433
|
+
variant: Some(wf_activation_job::Variant::RemoveFromCache(_)),
|
|
1434
|
+
}]
|
|
1435
|
+
);
|
|
1436
|
+
core.complete_workflow_activation(WfActivationCompletion::empty(TEST_Q, evict_act.run_id))
|
|
1437
|
+
.await
|
|
1438
|
+
.unwrap();
|
|
1439
|
+
|
|
1440
|
+
// Workflow starting over, this time issue the right command
|
|
1441
|
+
let act = core.poll_workflow_activation(TEST_Q).await.unwrap();
|
|
1442
|
+
core.complete_workflow_activation(WfActivationCompletion::from_cmds(
|
|
1443
|
+
TEST_Q,
|
|
1444
|
+
act.run_id,
|
|
1445
|
+
vec![StartTimer {
|
|
1446
|
+
seq: 1,
|
|
1447
|
+
..Default::default()
|
|
1448
|
+
}
|
|
1449
|
+
.into()],
|
|
1450
|
+
))
|
|
1451
|
+
.await
|
|
1452
|
+
.unwrap();
|
|
1453
|
+
let act = core.poll_workflow_activation(TEST_Q).await.unwrap();
|
|
1454
|
+
assert_matches!(
|
|
1455
|
+
act.jobs.as_slice(),
|
|
1456
|
+
[WfActivationJob {
|
|
1457
|
+
variant: Some(wf_activation_job::Variant::FireTimer(_)),
|
|
1458
|
+
},]
|
|
1459
|
+
);
|
|
1460
|
+
core.complete_workflow_activation(WfActivationCompletion::from_cmds(
|
|
1461
|
+
TEST_Q,
|
|
1462
|
+
act.run_id,
|
|
1463
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
|
1464
|
+
))
|
|
1465
|
+
.await
|
|
1466
|
+
.unwrap();
|
|
1467
|
+
core.shutdown().await;
|
|
1468
|
+
}
|
|
1469
|
+
|
|
1470
|
+
#[tokio::test]
|
|
1471
|
+
async fn poll_response_triggers_wf_error() {
|
|
1472
|
+
let mut t = TestHistoryBuilder::default();
|
|
1473
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
|
1474
|
+
// Add this nonsense event here to make applying the poll response fail
|
|
1475
|
+
t.add_external_signal_completed(100);
|
|
1476
|
+
t.add_full_wf_task();
|
|
1477
|
+
t.add_workflow_execution_completed();
|
|
1478
|
+
|
|
1479
|
+
let mut mh = MockPollCfg::from_resp_batches(
|
|
1480
|
+
"fake_wf_id",
|
|
1481
|
+
t,
|
|
1482
|
+
[ResponseType::AllHistory],
|
|
1483
|
+
MockServerGatewayApis::new(),
|
|
1484
|
+
);
|
|
1485
|
+
// Since applying the poll response immediately generates an error core will start polling again
|
|
1486
|
+
// Rather than panic on bad expectation we want to return the magic "no more work" error
|
|
1487
|
+
mh.enforce_correct_number_of_polls = false;
|
|
1488
|
+
let mock = build_mock_pollers(mh);
|
|
1489
|
+
let core = mock_core(mock);
|
|
1490
|
+
// Poll for first WFT, which is immediately an eviction
|
|
1491
|
+
let act = core.poll_workflow_activation(TEST_Q).await;
|
|
1492
|
+
assert_matches!(act, Err(PollWfError::TonicError(err))
|
|
1493
|
+
if err.message() == NO_MORE_WORK_ERROR_MSG);
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
// Verifies we can handle multiple wft timeouts in a row if lang is being very slow in responding
|
|
1497
|
+
#[tokio::test]
|
|
1498
|
+
async fn lang_slower_than_wft_timeouts() {
|
|
1499
|
+
let wfid = "fake_wf_id";
|
|
1500
|
+
let mut t = TestHistoryBuilder::default();
|
|
1501
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
|
1502
|
+
t.add_workflow_task_scheduled_and_started();
|
|
1503
|
+
t.add_workflow_task_timed_out();
|
|
1504
|
+
t.add_full_wf_task();
|
|
1505
|
+
t.add_workflow_execution_completed();
|
|
1506
|
+
|
|
1507
|
+
let tasks = [
|
|
1508
|
+
hist_to_poll_resp(&t, wfid.to_owned(), 1.into(), TEST_Q.to_string()),
|
|
1509
|
+
hist_to_poll_resp(&t, wfid.to_owned(), 1.into(), TEST_Q.to_string()),
|
|
1510
|
+
hist_to_poll_resp(&t, wfid.to_owned(), 1.into(), TEST_Q.to_string()),
|
|
1511
|
+
];
|
|
1512
|
+
let mut mock = MockServerGatewayApis::new();
|
|
1513
|
+
mock.expect_complete_workflow_task()
|
|
1514
|
+
.times(1)
|
|
1515
|
+
.returning(|_| Err(tonic::Status::not_found("Workflow task not found.")));
|
|
1516
|
+
mock.expect_complete_workflow_task()
|
|
1517
|
+
.times(1)
|
|
1518
|
+
.returning(|_| Ok(Default::default()));
|
|
1519
|
+
let mut mock = MocksHolder::from_gateway_with_responses(mock, tasks, []);
|
|
1520
|
+
mock.worker_cfg(TEST_Q, |wc| {
|
|
1521
|
+
wc.max_cached_workflows = 2;
|
|
1522
|
+
});
|
|
1523
|
+
let core = mock_core(mock);
|
|
1524
|
+
|
|
1525
|
+
let wf_task = core.poll_workflow_activation(TEST_Q).await.unwrap();
|
|
1526
|
+
let poll_until_no_work = core.poll_workflow_activation(TEST_Q).await;
|
|
1527
|
+
assert_matches!(poll_until_no_work, Err(PollWfError::TonicError(err))
|
|
1528
|
+
if err.message() == NO_MORE_WORK_ERROR_MSG);
|
|
1529
|
+
// This completion runs into a workflow task not found error, since it's completing a stale
|
|
1530
|
+
// task.
|
|
1531
|
+
core.complete_workflow_activation(WfActivationCompletion::empty(TEST_Q, wf_task.run_id))
|
|
1532
|
+
.await
|
|
1533
|
+
.unwrap();
|
|
1534
|
+
// Now we should get an eviction
|
|
1535
|
+
let wf_task = core.poll_workflow_activation(TEST_Q).await.unwrap();
|
|
1536
|
+
assert_matches!(
|
|
1537
|
+
wf_task.jobs.as_slice(),
|
|
1538
|
+
[WfActivationJob {
|
|
1539
|
+
variant: Some(wf_activation_job::Variant::RemoveFromCache(_)),
|
|
1540
|
+
}]
|
|
1541
|
+
);
|
|
1542
|
+
core.complete_workflow_activation(WfActivationCompletion::empty(TEST_Q, wf_task.run_id))
|
|
1543
|
+
.await
|
|
1544
|
+
.unwrap();
|
|
1545
|
+
// The last WFT buffered should be applied now
|
|
1546
|
+
let start_again = core.poll_workflow_activation(TEST_Q).await.unwrap();
|
|
1547
|
+
assert_matches!(
|
|
1548
|
+
start_again.jobs[0].variant,
|
|
1549
|
+
Some(wf_activation_job::Variant::StartWorkflow(_))
|
|
1550
|
+
);
|
|
1551
|
+
core.complete_workflow_activation(WfActivationCompletion::from_cmds(
|
|
1552
|
+
TEST_Q,
|
|
1553
|
+
start_again.run_id,
|
|
1554
|
+
vec![CompleteWorkflowExecution { result: None }.into()],
|
|
1555
|
+
))
|
|
1556
|
+
.await
|
|
1557
|
+
.unwrap();
|
|
1558
|
+
core.shutdown().await;
|
|
1559
|
+
}
|
|
1560
|
+
|
|
1561
|
+
#[tokio::test]
|
|
1562
|
+
async fn tries_cancel_of_completed_activity() {
|
|
1563
|
+
let mut t = TestHistoryBuilder::default();
|
|
1564
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
|
1565
|
+
t.add_full_wf_task();
|
|
1566
|
+
let scheduled_event_id = t.add_activity_task_scheduled("1");
|
|
1567
|
+
t.add_we_signaled("sig", vec![]);
|
|
1568
|
+
let started_event_id = t.add_activity_task_started(scheduled_event_id);
|
|
1569
|
+
t.add_activity_task_completed(scheduled_event_id, started_event_id, Default::default());
|
|
1570
|
+
t.add_workflow_task_scheduled_and_started();
|
|
1571
|
+
|
|
1572
|
+
let mock = MockServerGatewayApis::new();
|
|
1573
|
+
let mut mock = single_hist_mock_sg("fake_wf_id", t, &[1, 2], mock, true);
|
|
1574
|
+
mock.worker_cfg(TEST_Q, |cfg| cfg.max_cached_workflows = 1);
|
|
1575
|
+
let core = mock_core(mock);
|
|
1576
|
+
|
|
1577
|
+
let activation = core.poll_workflow_activation(TEST_Q).await.unwrap();
|
|
1578
|
+
core.complete_workflow_activation(WfActivationCompletion::from_cmd(
|
|
1579
|
+
TEST_Q,
|
|
1580
|
+
activation.run_id,
|
|
1581
|
+
ScheduleActivity {
|
|
1582
|
+
seq: 1,
|
|
1583
|
+
activity_id: "1".to_string(),
|
|
1584
|
+
..Default::default()
|
|
1585
|
+
}
|
|
1586
|
+
.into(),
|
|
1587
|
+
))
|
|
1588
|
+
.await
|
|
1589
|
+
.unwrap();
|
|
1590
|
+
let activation = core.poll_workflow_activation(TEST_Q).await.unwrap();
|
|
1591
|
+
assert_matches!(
|
|
1592
|
+
activation.jobs.as_slice(),
|
|
1593
|
+
[
|
|
1594
|
+
WfActivationJob {
|
|
1595
|
+
variant: Some(wf_activation_job::Variant::SignalWorkflow(_)),
|
|
1596
|
+
},
|
|
1597
|
+
WfActivationJob {
|
|
1598
|
+
variant: Some(wf_activation_job::Variant::ResolveActivity(_)),
|
|
1599
|
+
}
|
|
1600
|
+
]
|
|
1601
|
+
);
|
|
1602
|
+
core.complete_workflow_activation(WfActivationCompletion::from_cmds(
|
|
1603
|
+
TEST_Q,
|
|
1604
|
+
activation.run_id,
|
|
1605
|
+
vec![
|
|
1606
|
+
RequestCancelActivity { seq: 1 }.into(),
|
|
1607
|
+
CompleteWorkflowExecution { result: None }.into(),
|
|
1608
|
+
],
|
|
1609
|
+
))
|
|
1610
|
+
.await
|
|
1611
|
+
.unwrap();
|
|
1612
|
+
|
|
1613
|
+
core.shutdown().await;
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
#[tokio::test]
|
|
1617
|
+
async fn failing_wft_doesnt_eat_permit_forever() {
|
|
1618
|
+
let mut t = TestHistoryBuilder::default();
|
|
1619
|
+
t.add_by_type(EventType::WorkflowExecutionStarted);
|
|
1620
|
+
t.add_workflow_task_scheduled_and_started();
|
|
1621
|
+
|
|
1622
|
+
let failures = 5;
|
|
1623
|
+
// One extra response for when we stop failing
|
|
1624
|
+
let resps = (1..=(failures + 1)).map(|_| 1);
|
|
1625
|
+
let mock = MockServerGatewayApis::new();
|
|
1626
|
+
let mut mock = single_hist_mock_sg("fake_wf_id", t, resps, mock, true);
|
|
1627
|
+
mock.worker_cfg(TEST_Q, |cfg| {
|
|
1628
|
+
cfg.max_cached_workflows = 2;
|
|
1629
|
+
cfg.max_outstanding_workflow_tasks = 2;
|
|
1630
|
+
});
|
|
1631
|
+
let core = mock_core(mock);
|
|
1632
|
+
|
|
1633
|
+
// Spin failing the WFT to verify that we don't get stuck
|
|
1634
|
+
for _ in 1..=failures {
|
|
1635
|
+
let activation = core.poll_workflow_activation(TEST_Q).await.unwrap();
|
|
1636
|
+
// Issue a nonsense completion that will trigger a WFT failure
|
|
1637
|
+
core.complete_workflow_activation(WfActivationCompletion::from_cmd(
|
|
1638
|
+
TEST_Q,
|
|
1639
|
+
activation.run_id,
|
|
1640
|
+
RequestCancelActivity { seq: 1 }.into(),
|
|
1641
|
+
))
|
|
1642
|
+
.await
|
|
1643
|
+
.unwrap();
|
|
1644
|
+
let activation = core.poll_workflow_activation(TEST_Q).await.unwrap();
|
|
1645
|
+
assert_matches!(
|
|
1646
|
+
activation.jobs.as_slice(),
|
|
1647
|
+
[WfActivationJob {
|
|
1648
|
+
variant: Some(wf_activation_job::Variant::RemoveFromCache(_)),
|
|
1649
|
+
},]
|
|
1650
|
+
);
|
|
1651
|
+
core.complete_workflow_activation(WfActivationCompletion::empty(TEST_Q, activation.run_id))
|
|
1652
|
+
.await
|
|
1653
|
+
.unwrap();
|
|
1654
|
+
assert_eq!(core.outstanding_wfts(TEST_Q), 0);
|
|
1655
|
+
assert_eq!(core.available_wft_permits(TEST_Q), 2);
|
|
1656
|
+
}
|
|
1657
|
+
let activation = core.poll_workflow_activation(TEST_Q).await.unwrap();
|
|
1658
|
+
core.complete_workflow_activation(WfActivationCompletion::from_cmd(
|
|
1659
|
+
TEST_Q,
|
|
1660
|
+
activation.run_id,
|
|
1661
|
+
CompleteWorkflowExecution { result: None }.into(),
|
|
1662
|
+
))
|
|
1663
|
+
.await
|
|
1664
|
+
.unwrap();
|
|
1665
|
+
|
|
1666
|
+
core.shutdown().await;
|
|
1667
|
+
}
|
package/sdk-core/src/errors.rs
CHANGED
|
@@ -16,6 +16,23 @@ pub(crate) struct WorkflowUpdateError {
|
|
|
16
16
|
pub task_token: Option<TaskToken>,
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
+
impl From<WorkflowMissingError> for WorkflowUpdateError {
|
|
20
|
+
fn from(wme: WorkflowMissingError) -> Self {
|
|
21
|
+
Self {
|
|
22
|
+
source: WFMachinesError::Fatal("Workflow machines missing".to_string()),
|
|
23
|
+
run_id: wme.run_id,
|
|
24
|
+
task_token: None,
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/// The workflow machines were expected to be in the cache but were not
|
|
30
|
+
#[derive(Debug)]
|
|
31
|
+
pub(crate) struct WorkflowMissingError {
|
|
32
|
+
/// The run id of the erring workflow
|
|
33
|
+
pub run_id: String,
|
|
34
|
+
}
|
|
35
|
+
|
|
19
36
|
/// Errors thrown during initialization of [crate::Core]
|
|
20
37
|
#[derive(thiserror::Error, Debug)]
|
|
21
38
|
pub enum CoreInitError {
|
|
@@ -33,15 +50,6 @@ pub enum CoreInitError {
|
|
|
33
50
|
/// Errors thrown by [crate::Core::poll_workflow_activation]
|
|
34
51
|
#[derive(thiserror::Error, Debug)]
|
|
35
52
|
pub enum PollWfError {
|
|
36
|
-
/// There was an error specific to a workflow instance. The cached workflow should be deleted
|
|
37
|
-
/// from lang side.
|
|
38
|
-
#[error("There was an error with the workflow instance with run id ({run_id}): {source:?}")]
|
|
39
|
-
WorkflowUpdateError {
|
|
40
|
-
/// Underlying workflow error
|
|
41
|
-
source: anyhow::Error,
|
|
42
|
-
/// The run id of the erring workflow
|
|
43
|
-
run_id: String,
|
|
44
|
-
},
|
|
45
53
|
/// [crate::Core::shutdown] was called, and there are no more replay tasks to be handled. Lang
|
|
46
54
|
/// must call [crate::Core::complete_workflow_activation] for any remaining tasks, and then may
|
|
47
55
|
/// exit.
|
|
@@ -61,20 +69,11 @@ pub enum PollWfError {
|
|
|
61
69
|
NoWorkerForQueue(String),
|
|
62
70
|
}
|
|
63
71
|
|
|
64
|
-
impl From<WorkflowUpdateError> for PollWfError {
|
|
65
|
-
fn from(e: WorkflowUpdateError) -> Self {
|
|
66
|
-
Self::WorkflowUpdateError {
|
|
67
|
-
source: e.source.into(),
|
|
68
|
-
run_id: e.run_id,
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
|
|
73
72
|
impl From<WorkerLookupErr> for PollWfError {
|
|
74
73
|
fn from(e: WorkerLookupErr) -> Self {
|
|
75
74
|
match e {
|
|
76
|
-
WorkerLookupErr::Shutdown(_) =>
|
|
77
|
-
WorkerLookupErr::NoWorker(s) =>
|
|
75
|
+
WorkerLookupErr::Shutdown(_) => Self::ShutDown,
|
|
76
|
+
WorkerLookupErr::NoWorker(s) => Self::NoWorkerForQueue(s),
|
|
78
77
|
}
|
|
79
78
|
}
|
|
80
79
|
}
|
|
@@ -98,8 +97,8 @@ pub enum PollActivityError {
|
|
|
98
97
|
impl From<WorkerLookupErr> for PollActivityError {
|
|
99
98
|
fn from(e: WorkerLookupErr) -> Self {
|
|
100
99
|
match e {
|
|
101
|
-
WorkerLookupErr::Shutdown(_) =>
|
|
102
|
-
WorkerLookupErr::NoWorker(s) =>
|
|
100
|
+
WorkerLookupErr::Shutdown(_) => Self::ShutDown,
|
|
101
|
+
WorkerLookupErr::NoWorker(s) => Self::NoWorkerForQueue(s),
|
|
103
102
|
}
|
|
104
103
|
}
|
|
105
104
|
}
|
|
@@ -116,15 +115,6 @@ pub enum CompleteWfError {
|
|
|
116
115
|
/// The completion, which may not be included to avoid unnecessary copies.
|
|
117
116
|
completion: Option<WfActivationCompletion>,
|
|
118
117
|
},
|
|
119
|
-
/// There was an error specific to a workflow instance. The cached workflow should be deleted
|
|
120
|
-
/// from lang side.
|
|
121
|
-
#[error("There was an error with the workflow instance with run id ({run_id}): {source:?}")]
|
|
122
|
-
WorkflowUpdateError {
|
|
123
|
-
/// Underlying workflow error
|
|
124
|
-
source: anyhow::Error,
|
|
125
|
-
/// The run id of the erring workflow
|
|
126
|
-
run_id: String,
|
|
127
|
-
},
|
|
128
118
|
/// There is no worker registered for the queue being polled
|
|
129
119
|
#[error("No worker registered for queue: {0}")]
|
|
130
120
|
NoWorkerForQueue(String),
|
|
@@ -134,20 +124,12 @@ pub enum CompleteWfError {
|
|
|
134
124
|
TonicError(#[from] tonic::Status),
|
|
135
125
|
}
|
|
136
126
|
|
|
137
|
-
impl From<WorkflowUpdateError> for CompleteWfError {
|
|
138
|
-
fn from(e: WorkflowUpdateError) -> Self {
|
|
139
|
-
Self::WorkflowUpdateError {
|
|
140
|
-
source: e.source.into(),
|
|
141
|
-
run_id: e.run_id,
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
|
|
146
127
|
impl From<WorkerLookupErr> for CompleteWfError {
|
|
147
128
|
fn from(e: WorkerLookupErr) -> Self {
|
|
148
129
|
match e {
|
|
149
|
-
WorkerLookupErr::Shutdown(s)
|
|
150
|
-
|
|
130
|
+
WorkerLookupErr::Shutdown(s) | WorkerLookupErr::NoWorker(s) => {
|
|
131
|
+
Self::NoWorkerForQueue(s)
|
|
132
|
+
}
|
|
151
133
|
}
|
|
152
134
|
}
|
|
153
135
|
}
|
|
@@ -175,8 +157,9 @@ pub enum CompleteActivityError {
|
|
|
175
157
|
impl From<WorkerLookupErr> for CompleteActivityError {
|
|
176
158
|
fn from(e: WorkerLookupErr) -> Self {
|
|
177
159
|
match e {
|
|
178
|
-
WorkerLookupErr::Shutdown(s)
|
|
179
|
-
|
|
160
|
+
WorkerLookupErr::Shutdown(s) | WorkerLookupErr::NoWorker(s) => {
|
|
161
|
+
Self::NoWorkerForQueue(s)
|
|
162
|
+
}
|
|
180
163
|
}
|
|
181
164
|
}
|
|
182
165
|
}
|
package/sdk-core/src/lib.rs
CHANGED
|
@@ -202,7 +202,7 @@ pub struct CoreInitOptions {
|
|
|
202
202
|
|
|
203
203
|
/// Initializes an instance of the core sdk and establishes a connection to the temporal server.
|
|
204
204
|
///
|
|
205
|
-
/// Note: Also creates a tokio runtime that will be used for all client-server interactions.
|
|
205
|
+
/// Note: Also creates a tokio runtime that will be used for all client-server interactions.
|
|
206
206
|
///
|
|
207
207
|
/// # Panics
|
|
208
208
|
/// * Will panic if called from within an async context, as it will construct a runtime and you
|
|
@@ -272,7 +272,8 @@ impl Core for CoreSDK {
|
|
|
272
272
|
}
|
|
273
273
|
}
|
|
274
274
|
|
|
275
|
-
#[instrument(level = "debug", skip(self, completion),
|
|
275
|
+
#[instrument(level = "debug", skip(self, completion),
|
|
276
|
+
fields(completion=%&completion, run_id=%completion.run_id))]
|
|
276
277
|
async fn complete_workflow_activation(
|
|
277
278
|
&self,
|
|
278
279
|
completion: WfActivationCompletion,
|
|
@@ -308,7 +309,7 @@ impl Core for CoreSDK {
|
|
|
308
309
|
|
|
309
310
|
fn request_workflow_eviction(&self, task_queue: &str, run_id: &str) {
|
|
310
311
|
if let Ok(w) = self.worker(task_queue) {
|
|
311
|
-
w.request_wf_eviction(run_id);
|
|
312
|
+
w.request_wf_eviction(run_id, "Eviction explicitly requested by lang");
|
|
312
313
|
}
|
|
313
314
|
}
|
|
314
315
|
|
|
@@ -366,6 +367,15 @@ impl CoreSDK {
|
|
|
366
367
|
self.workers.set_worker_for_task_queue(tq, worker).unwrap();
|
|
367
368
|
}
|
|
368
369
|
|
|
370
|
+
#[cfg(test)]
|
|
371
|
+
pub(crate) fn outstanding_wfts(&self, tq: &str) -> usize {
|
|
372
|
+
self.worker(tq).unwrap().outstanding_workflow_tasks()
|
|
373
|
+
}
|
|
374
|
+
#[cfg(test)]
|
|
375
|
+
pub(crate) fn available_wft_permits(&self, tq: &str) -> usize {
|
|
376
|
+
self.worker(tq).unwrap().available_wft_permits()
|
|
377
|
+
}
|
|
378
|
+
|
|
369
379
|
fn get_sticky_q_name_for_worker(&self, config: &WorkerConfig) -> Option<String> {
|
|
370
380
|
if config.max_cached_workflows > 0 {
|
|
371
381
|
Some(format!(
|
|
@@ -128,7 +128,7 @@ impl ActivityMachine {
|
|
|
128
128
|
{
|
|
129
129
|
r.push(MachineResponse::PushWFJob(
|
|
130
130
|
self.create_cancelation_resolve(None).into(),
|
|
131
|
-
))
|
|
131
|
+
));
|
|
132
132
|
}
|
|
133
133
|
r
|
|
134
134
|
}
|
|
@@ -305,6 +305,20 @@ impl TryFrom<CommandType> for ActivityMachineEvents {
|
|
|
305
305
|
|
|
306
306
|
impl Cancellable for ActivityMachine {
|
|
307
307
|
fn cancel(&mut self) -> Result<Vec<MachineResponse>, MachineError<Self::Error>> {
|
|
308
|
+
if matches!(
|
|
309
|
+
self.state(),
|
|
310
|
+
ActivityMachineState::Completed(_)
|
|
311
|
+
| ActivityMachineState::Canceled(_)
|
|
312
|
+
| ActivityMachineState::Failed(_)
|
|
313
|
+
| ActivityMachineState::TimedOut(_)
|
|
314
|
+
) {
|
|
315
|
+
// Ignore attempted cancels in terminal states
|
|
316
|
+
debug!(
|
|
317
|
+
"Attempted to cancel already resolved activity (seq {})",
|
|
318
|
+
self.shared_state.attrs.seq
|
|
319
|
+
);
|
|
320
|
+
return Ok(vec![]);
|
|
321
|
+
}
|
|
308
322
|
let event = match self.shared_state.cancellation_type {
|
|
309
323
|
ActivityCancellationType::Abandon => ActivityMachineEvents::Abandon,
|
|
310
324
|
_ => ActivityMachineEvents::Cancel,
|
|
@@ -316,9 +330,15 @@ impl Cancellable for ActivityMachine {
|
|
|
316
330
|
ActivityMachineCommand::RequestCancellation(cmd) => {
|
|
317
331
|
self.machine_responses_from_cancel_request(cmd)
|
|
318
332
|
}
|
|
319
|
-
ActivityMachineCommand::Cancel(
|
|
320
|
-
|
|
321
|
-
|
|
333
|
+
ActivityMachineCommand::Cancel(details) => {
|
|
334
|
+
vec![self
|
|
335
|
+
.create_cancelation_resolve(
|
|
336
|
+
details
|
|
337
|
+
.map(TryInto::try_into)
|
|
338
|
+
.transpose()
|
|
339
|
+
.unwrap_or_default(),
|
|
340
|
+
)
|
|
341
|
+
.into()]
|
|
322
342
|
}
|
|
323
343
|
x => panic!("Invalid cancel event response {:?}", x),
|
|
324
344
|
})
|
|
@@ -665,7 +685,7 @@ fn notify_lang_activity_cancelled(
|
|
|
665
685
|
) -> ActivityMachineTransition<Canceled> {
|
|
666
686
|
ActivityMachineTransition::ok_shared(
|
|
667
687
|
vec![ActivityMachineCommand::Cancel(
|
|
668
|
-
canceled_event.
|
|
688
|
+
canceled_event.and_then(|e| e.details),
|
|
669
689
|
)],
|
|
670
690
|
Canceled::default(),
|
|
671
691
|
dat,
|
|
@@ -735,6 +755,7 @@ mod test {
|
|
|
735
755
|
workflow::managed_wf::ManagedWFFunc,
|
|
736
756
|
};
|
|
737
757
|
use rstest::{fixture, rstest};
|
|
758
|
+
use std::mem::discriminant;
|
|
738
759
|
use temporal_sdk_core_protos::coresdk::workflow_activation::{
|
|
739
760
|
wf_activation_job, WfActivationJob,
|
|
740
761
|
};
|
|
@@ -835,4 +856,22 @@ mod test {
|
|
|
835
856
|
);
|
|
836
857
|
wfm.shutdown().await.unwrap();
|
|
837
858
|
}
|
|
859
|
+
|
|
860
|
+
#[test]
|
|
861
|
+
fn cancels_ignored_terminal() {
|
|
862
|
+
for state in [
|
|
863
|
+
ActivityMachineState::Canceled(Canceled {}),
|
|
864
|
+
Failed {}.into(),
|
|
865
|
+
TimedOut {}.into(),
|
|
866
|
+
Completed {}.into(),
|
|
867
|
+
] {
|
|
868
|
+
let mut s = ActivityMachine {
|
|
869
|
+
state: state.clone(),
|
|
870
|
+
shared_state: Default::default(),
|
|
871
|
+
};
|
|
872
|
+
let cmds = s.cancel().unwrap();
|
|
873
|
+
assert_eq!(cmds.len(), 0);
|
|
874
|
+
assert_eq!(discriminant(&state), discriminant(&s.state));
|
|
875
|
+
}
|
|
876
|
+
}
|
|
838
877
|
}
|
|
@@ -54,6 +54,12 @@ fsm! {
|
|
|
54
54
|
Started --(ChildWorkflowExecutionTimedOut(i32), shared on_child_workflow_execution_timed_out) --> TimedOut;
|
|
55
55
|
Started --(ChildWorkflowExecutionCancelled, shared on_child_workflow_execution_cancelled) --> Cancelled;
|
|
56
56
|
Started --(ChildWorkflowExecutionTerminated, shared on_child_workflow_execution_terminated) --> Terminated;
|
|
57
|
+
|
|
58
|
+
// Ignore any spurious cancellations after resolution
|
|
59
|
+
Cancelled --(Cancel) --> Cancelled;
|
|
60
|
+
Failed --(Cancel) --> Failed;
|
|
61
|
+
TimedOut --(Cancel) --> TimedOut;
|
|
62
|
+
Completed --(Cancel) --> Completed;
|
|
57
63
|
}
|
|
58
64
|
|
|
59
65
|
pub struct ChildWorkflowExecutionStartedEvent {
|
|
@@ -288,7 +294,7 @@ pub(super) struct Terminated {}
|
|
|
288
294
|
#[derive(Default, Clone)]
|
|
289
295
|
pub(super) struct TimedOut {}
|
|
290
296
|
|
|
291
|
-
#[derive(Default, Clone)]
|
|
297
|
+
#[derive(Default, Clone, Debug)]
|
|
292
298
|
pub(super) struct SharedState {
|
|
293
299
|
initiated_event_id: i64,
|
|
294
300
|
started_event_id: i64,
|
|
@@ -630,6 +636,7 @@ mod test {
|
|
|
630
636
|
};
|
|
631
637
|
use anyhow::anyhow;
|
|
632
638
|
use rstest::{fixture, rstest};
|
|
639
|
+
use std::mem::discriminant;
|
|
633
640
|
use temporal_sdk_core_protos::coresdk::{
|
|
634
641
|
child_workflow::child_workflow_result,
|
|
635
642
|
workflow_activation::resolve_child_workflow_execution_start::Status as StartStatus,
|
|
@@ -643,11 +650,11 @@ mod test {
|
|
|
643
650
|
}
|
|
644
651
|
|
|
645
652
|
impl Expectation {
|
|
646
|
-
fn try_from_u8(x: u8) -> Option<Self> {
|
|
653
|
+
const fn try_from_u8(x: u8) -> Option<Self> {
|
|
647
654
|
Some(match x {
|
|
648
|
-
0 =>
|
|
649
|
-
1 =>
|
|
650
|
-
2 =>
|
|
655
|
+
0 => Self::Success,
|
|
656
|
+
1 => Self::Failure,
|
|
657
|
+
2 => Self::StartFailure,
|
|
651
658
|
_ => return None,
|
|
652
659
|
})
|
|
653
660
|
}
|
|
@@ -693,12 +700,7 @@ mod test {
|
|
|
693
700
|
};
|
|
694
701
|
match (
|
|
695
702
|
expectation,
|
|
696
|
-
start_res
|
|
697
|
-
.as_started()
|
|
698
|
-
.unwrap()
|
|
699
|
-
.result(&mut ctx)
|
|
700
|
-
.await
|
|
701
|
-
.status,
|
|
703
|
+
start_res.into_started().unwrap().result().await.status,
|
|
702
704
|
) {
|
|
703
705
|
(Expectation::Success, Some(child_workflow_result::Status::Completed(_))) => {
|
|
704
706
|
Ok(().into())
|
|
@@ -803,4 +805,22 @@ mod test {
|
|
|
803
805
|
);
|
|
804
806
|
wfm.shutdown().await.unwrap();
|
|
805
807
|
}
|
|
808
|
+
|
|
809
|
+
#[test]
|
|
810
|
+
fn cancels_ignored_terminal() {
|
|
811
|
+
for state in [
|
|
812
|
+
ChildWorkflowMachineState::Cancelled(Cancelled {}),
|
|
813
|
+
Failed {}.into(),
|
|
814
|
+
TimedOut {}.into(),
|
|
815
|
+
Completed {}.into(),
|
|
816
|
+
] {
|
|
817
|
+
let mut s = ChildWorkflowMachine {
|
|
818
|
+
state: state.clone(),
|
|
819
|
+
shared_state: Default::default(),
|
|
820
|
+
};
|
|
821
|
+
let cmds = s.cancel().unwrap();
|
|
822
|
+
assert_eq!(cmds.len(), 0);
|
|
823
|
+
assert_eq!(discriminant(&state), discriminant(&s.state));
|
|
824
|
+
}
|
|
825
|
+
}
|
|
806
826
|
}
|
|
@@ -117,7 +117,7 @@ pub(super) struct CompleteWorkflowCommandRecorded {}
|
|
|
117
117
|
|
|
118
118
|
impl From<CompleteWorkflowCommandCreated> for CompleteWorkflowCommandRecorded {
|
|
119
119
|
fn from(_: CompleteWorkflowCommandCreated) -> Self {
|
|
120
|
-
|
|
120
|
+
Self::default()
|
|
121
121
|
}
|
|
122
122
|
}
|
|
123
123
|
|