@pgflow/core 0.0.5-prealpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/LICENSE.md +660 -0
  2. package/README.md +373 -0
  3. package/__tests__/mocks/index.ts +1 -0
  4. package/__tests__/mocks/postgres.ts +37 -0
  5. package/__tests__/types/PgflowSqlClient.test-d.ts +59 -0
  6. package/dist/LICENSE.md +660 -0
  7. package/dist/README.md +373 -0
  8. package/dist/index.js +54 -0
  9. package/docs/options_for_flow_and_steps.md +75 -0
  10. package/docs/pgflow-blob-reference-system.md +179 -0
  11. package/eslint.config.cjs +22 -0
  12. package/example-flow.mermaid +5 -0
  13. package/example-flow.svg +1 -0
  14. package/flow-lifecycle.mermaid +83 -0
  15. package/flow-lifecycle.svg +1 -0
  16. package/out-tsc/vitest/__tests__/mocks/index.d.ts +2 -0
  17. package/out-tsc/vitest/__tests__/mocks/index.d.ts.map +1 -0
  18. package/out-tsc/vitest/__tests__/mocks/postgres.d.ts +15 -0
  19. package/out-tsc/vitest/__tests__/mocks/postgres.d.ts.map +1 -0
  20. package/out-tsc/vitest/__tests__/types/PgflowSqlClient.test-d.d.ts +2 -0
  21. package/out-tsc/vitest/__tests__/types/PgflowSqlClient.test-d.d.ts.map +1 -0
  22. package/out-tsc/vitest/tsconfig.spec.tsbuildinfo +1 -0
  23. package/out-tsc/vitest/vite.config.d.ts +3 -0
  24. package/out-tsc/vitest/vite.config.d.ts.map +1 -0
  25. package/package.json +28 -0
  26. package/pkgs/core/dist/index.js +54 -0
  27. package/pkgs/core/dist/pkgs/core/LICENSE.md +660 -0
  28. package/pkgs/core/dist/pkgs/core/README.md +373 -0
  29. package/pkgs/dsl/dist/index.js +123 -0
  30. package/pkgs/dsl/dist/pkgs/dsl/README.md +11 -0
  31. package/project.json +125 -0
  32. package/prompts/architect.md +87 -0
  33. package/prompts/condition.md +33 -0
  34. package/prompts/declarative_sql.md +15 -0
  35. package/prompts/deps_in_payloads.md +20 -0
  36. package/prompts/dsl-multi-arg.ts +48 -0
  37. package/prompts/dsl-options.md +39 -0
  38. package/prompts/dsl-single-arg.ts +51 -0
  39. package/prompts/dsl-two-arg.ts +61 -0
  40. package/prompts/dsl.md +119 -0
  41. package/prompts/fanout_steps.md +1 -0
  42. package/prompts/json_schemas.md +36 -0
  43. package/prompts/one_shot.md +286 -0
  44. package/prompts/pgtap.md +229 -0
  45. package/prompts/sdk.md +59 -0
  46. package/prompts/step_types.md +62 -0
  47. package/prompts/versioning.md +16 -0
  48. package/queries/fail_permanently.sql +17 -0
  49. package/queries/fail_task.sql +21 -0
  50. package/queries/sequential.sql +47 -0
  51. package/queries/two_roots_left_right.sql +59 -0
  52. package/schema.svg +1 -0
  53. package/scripts/colorize-pgtap-output.awk +72 -0
  54. package/scripts/run-test-with-colors +5 -0
  55. package/scripts/watch-test +7 -0
  56. package/src/PgflowSqlClient.ts +85 -0
  57. package/src/database-types.ts +759 -0
  58. package/src/index.ts +3 -0
  59. package/src/types.ts +103 -0
  60. package/supabase/config.toml +32 -0
  61. package/supabase/migrations/000000_schema.sql +150 -0
  62. package/supabase/migrations/000005_create_flow.sql +29 -0
  63. package/supabase/migrations/000010_add_step.sql +48 -0
  64. package/supabase/migrations/000015_start_ready_steps.sql +45 -0
  65. package/supabase/migrations/000020_start_flow.sql +46 -0
  66. package/supabase/migrations/000030_read_with_poll_backport.sql +70 -0
  67. package/supabase/migrations/000040_poll_for_tasks.sql +100 -0
  68. package/supabase/migrations/000045_maybe_complete_run.sql +30 -0
  69. package/supabase/migrations/000050_complete_task.sql +98 -0
  70. package/supabase/migrations/000055_calculate_retry_delay.sql +11 -0
  71. package/supabase/migrations/000060_fail_task.sql +124 -0
  72. package/supabase/migrations/000_edge_worker_initial.sql +86 -0
  73. package/supabase/seed.sql +202 -0
  74. package/supabase/tests/add_step/basic_step_addition.test.sql +29 -0
  75. package/supabase/tests/add_step/circular_dependency.test.sql +21 -0
  76. package/supabase/tests/add_step/flow_isolation.test.sql +26 -0
  77. package/supabase/tests/add_step/idempotent_step_addition.test.sql +20 -0
  78. package/supabase/tests/add_step/invalid_step_slug.test.sql +16 -0
  79. package/supabase/tests/add_step/nonexistent_dependency.test.sql +16 -0
  80. package/supabase/tests/add_step/nonexistent_flow.test.sql +13 -0
  81. package/supabase/tests/add_step/options.test.sql +66 -0
  82. package/supabase/tests/add_step/step_with_dependency.test.sql +36 -0
  83. package/supabase/tests/add_step/step_with_multiple_dependencies.test.sql +46 -0
  84. package/supabase/tests/complete_task/archives_message.test.sql +67 -0
  85. package/supabase/tests/complete_task/completes_run_if_no_more_remaining_steps.test.sql +62 -0
  86. package/supabase/tests/complete_task/completes_task_and_updates_dependents.test.sql +64 -0
  87. package/supabase/tests/complete_task/decrements_remaining_steps_if_completing_step.test.sql +62 -0
  88. package/supabase/tests/complete_task/saves_output_when_completing_run.test.sql +57 -0
  89. package/supabase/tests/create_flow/flow_creation.test.sql +27 -0
  90. package/supabase/tests/create_flow/idempotency_and_duplicates.test.sql +26 -0
  91. package/supabase/tests/create_flow/invalid_slug.test.sql +13 -0
  92. package/supabase/tests/create_flow/options.test.sql +57 -0
  93. package/supabase/tests/fail_task/exponential_backoff.test.sql +70 -0
  94. package/supabase/tests/fail_task/mark_as_failed_if_no_retries_available.test.sql +49 -0
  95. package/supabase/tests/fail_task/respects_flow_retry_settings.test.sql +48 -0
  96. package/supabase/tests/fail_task/respects_step_retry_settings.test.sql +48 -0
  97. package/supabase/tests/fail_task/retry_task_if_retries_available.test.sql +39 -0
  98. package/supabase/tests/is_valid_slug.test.sql +72 -0
  99. package/supabase/tests/poll_for_tasks/builds_proper_input_from_deps_outputs.test.sql +35 -0
  100. package/supabase/tests/poll_for_tasks/hides_messages.test.sql +35 -0
  101. package/supabase/tests/poll_for_tasks/increments_attempts_count.test.sql +35 -0
  102. package/supabase/tests/poll_for_tasks/multiple_task_processing.test.sql +24 -0
  103. package/supabase/tests/poll_for_tasks/polls_only_queued_tasks.test.sql +35 -0
  104. package/supabase/tests/poll_for_tasks/reads_messages.test.sql +38 -0
  105. package/supabase/tests/poll_for_tasks/returns_no_tasks_if_no_step_task_for_message.test.sql +34 -0
  106. package/supabase/tests/poll_for_tasks/returns_no_tasks_if_queue_is_empty.test.sql +19 -0
  107. package/supabase/tests/poll_for_tasks/returns_no_tasks_when_qty_set_to_0.test.sql +22 -0
  108. package/supabase/tests/poll_for_tasks/sets_vt_delay_based_on_opt_timeout.test.sql +41 -0
  109. package/supabase/tests/poll_for_tasks/tasks_reapppear_if_not_processed_in_time.test.sql +59 -0
  110. package/supabase/tests/start_flow/creates_run.test.sql +24 -0
  111. package/supabase/tests/start_flow/creates_step_states_for_all_steps.test.sql +25 -0
  112. package/supabase/tests/start_flow/creates_step_tasks_only_for_root_steps.test.sql +54 -0
  113. package/supabase/tests/start_flow/returns_run.test.sql +24 -0
  114. package/supabase/tests/start_flow/sends_messages_on_the_queue.test.sql +50 -0
  115. package/supabase/tests/start_flow/starts_only_root_steps.test.sql +21 -0
  116. package/supabase/tests/step_dsl_is_idempotent.test.sql +34 -0
  117. package/tsconfig.json +16 -0
  118. package/tsconfig.lib.json +26 -0
  119. package/tsconfig.spec.json +35 -0
  120. package/vite.config.ts +57 -0
@@ -0,0 +1,25 @@
1
+ begin;
2
+ select plan(2);
3
+ select pgflow_tests.reset_db();
4
+ select pgflow_tests.setup_flow('sequential');
5
+ select pgflow_tests.setup_flow('sequential_other');
6
+
7
+ -- SETUP: Start a flow run
8
+ select pgflow.start_flow('sequential', '"hello"'::jsonb);
9
+
10
+ -- TEST: All step states from flow should be created
11
+ select set_eq(
12
+ $$ SELECT step_slug
13
+ FROM pgflow.step_states WHERE flow_slug = 'sequential' $$,
14
+ array['first', 'second', 'last']::text [],
15
+ 'All step states from flow should be created'
16
+ );
17
+
18
+ -- TEST: No steps from other flows should be created
19
+ select is_empty(
20
+ $$ SELECT * FROM pgflow.step_states WHERE flow_slug <> 'sequential' $$,
21
+ 'No steps from other flows should be created'
22
+ );
23
+
24
+ select finish();
25
+ rollback;
@@ -0,0 +1,54 @@
1
+ begin;
2
+ select plan(5);
3
+ select pgflow_tests.reset_db();
4
+ select pgflow_tests.setup_flow('two_roots');
5
+
6
+ select pgflow.start_flow('two_roots', '"hello"'::jsonb);
7
+
8
+ -- TEST: A step_task record should be created only for the root step
9
+ select is(
10
+ (
11
+ select
12
+ array_agg(
13
+ step_slug
14
+ order by step_slug
15
+ )
16
+ from pgflow.step_tasks
17
+ where flow_slug = 'two_roots'
18
+ ),
19
+ array['root_a', 'root_b']::text [],
20
+ 'A step_task record should be created for each root step'
21
+ );
22
+
23
+ -- TEST: Two messages should be in the queue, one per each root step
24
+ select is(
25
+ (select count(*)::int from pgmq.q_two_roots),
26
+ 2::int,
27
+ 'Two messages should be in the queue, one per each root step'
28
+ );
29
+
30
+ -- TEST; Messages have appropriate flow slugs
31
+ select is(
32
+ (select array_agg(message ->> 'flow_slug') from pgmq.q_two_roots),
33
+ array['two_roots', 'two_roots'],
34
+ 'Messages have appropriate flow slugs'
35
+ );
36
+
37
+ -- TEST: Messages have appropriate step slugs
38
+ select is(
39
+ (select array_agg(message ->> 'step_slug') from pgmq.q_two_roots),
40
+ array['root_a', 'root_b']::text [],
41
+ 'Messages have appropriate step slugs'
42
+ );
43
+
44
+ select is(
45
+ (select array_agg(message ->> 'run_id') from pgmq.q_two_roots),
46
+ (
47
+ select array_agg(run_id::text) from pgflow.step_tasks
48
+ where flow_slug = 'two_roots'
49
+ ),
50
+ 'Messages have appropriate run_ids'
51
+ );
52
+
53
+ select finish();
54
+ rollback;
@@ -0,0 +1,24 @@
1
+ begin;
2
+ select plan(2);
3
+ select pgflow_tests.reset_db();
4
+ select pgflow_tests.setup_flow('sequential');
5
+ select pgflow_tests.setup_flow('two_roots');
6
+
7
+ -- TEST: start_flow() returns started step states
8
+ select results_eq(
9
+ $$ SELECT flow_slug, status, input
10
+ FROM pgflow.start_flow('sequential', '"hello"'::jsonb) $$,
11
+ $$ VALUES ('sequential', 'started', '"hello"'::jsonb) $$,
12
+ 'start_flow() should return a run'
13
+ );
14
+
15
+ -- TEST: start_flow() returns started step states
16
+ select results_eq(
17
+ $$ SELECT flow_slug, status, input
18
+ FROM pgflow.start_flow('sequential', '"world"'::jsonb) $$,
19
+ $$ VALUES ('sequential', 'started', '"world"'::jsonb) $$,
20
+ 'start_flow() should return a single run even for flow that have two root steps'
21
+ );
22
+
23
+ select finish();
24
+ rollback;
@@ -0,0 +1,50 @@
1
+ begin;
2
+ select plan(2);
3
+ select pgflow_tests.reset_db();
4
+ select pgflow_tests.setup_flow('sequential');
5
+
6
+ select pgflow.start_flow('sequential', '"hello"'::jsonb);
7
+
8
+ -- TEST: A step_task record should be created only for the root step
9
+ select is(
10
+ (
11
+ select
12
+ array_agg(
13
+ step_slug
14
+ order by step_slug
15
+ )
16
+ from pgflow.step_tasks
17
+ where flow_slug = 'sequential'
18
+ ),
19
+ array['first']::text [],
20
+ 'A step_task record should be created only for the root step'
21
+ );
22
+
23
+ -- TEST: The message in the queue should contain the correct step info
24
+ select is(
25
+ (
26
+ select q.message
27
+ from pgflow.step_tasks as st
28
+ inner join pgmq.q_sequential as q
29
+ on st.message_id = q.msg_id
30
+ where
31
+ st.flow_slug = 'sequential'
32
+ and st.step_slug = 'first'
33
+ ),
34
+ jsonb_build_object(
35
+ 'flow_slug', 'sequential',
36
+ 'run_id', (
37
+ select run_id
38
+ from pgflow.step_tasks
39
+ where
40
+ flow_slug = 'sequential'
41
+ and step_slug = 'first'
42
+ ),
43
+ 'step_slug', 'first',
44
+ 'task_index', 0
45
+ ),
46
+ 'The message in the queue should contain the correct step info'
47
+ );
48
+
49
+ select finish();
50
+ rollback;
@@ -0,0 +1,21 @@
1
+ begin;
2
+ select plan(1);
3
+ select pgflow_tests.reset_db();
4
+ select pgflow_tests.setup_flow('sequential');
5
+
6
+ -- SETUP: Start a flow run
7
+ select pgflow.start_flow('sequential', '"hello"'::jsonb);
8
+
9
+ -- TEST: Only root steps should be started
10
+ -- (Root steps are steps with no dependencies)
11
+ select results_eq(
12
+ $$ SELECT step_slug
13
+ FROM pgflow.step_states
14
+ WHERE flow_slug = 'sequential'
15
+ AND status = 'started' $$,
16
+ $$ VALUES ('first') $$,
17
+ 'Only root steps should be started'
18
+ );
19
+
20
+ select finish();
21
+ rollback;
@@ -0,0 +1,34 @@
1
+ begin;
2
+ select plan(2);
3
+ select pgflow_tests.reset_db();
4
+
5
+ -- Create a simple flow first
6
+ select pgflow.create_flow('test_flow');
7
+ select pgflow.add_step('test_flow', 'step_one');
8
+ select pgflow.add_step('test_flow', 'step_two', array['step_one']);
9
+ select
10
+ pgflow.add_step('test_flow', 'step_three', array['step_one', 'step_two']);
11
+
12
+ -- Run the same flow creation and step addition sequence again to verify those are idempotent
13
+ select pgflow.create_flow('test_flow');
14
+ select pgflow.add_step('test_flow', 'step_one');
15
+ select pgflow.add_step('test_flow', 'step_two', array['step_one']);
16
+ select
17
+ pgflow.add_step('test_flow', 'step_three', array['step_one', 'step_two']);
18
+
19
+ -- Check the state after first sequence
20
+ select results_eq(
21
+ $$ SELECT step_slug FROM pgflow.steps WHERE flow_slug = 'test_flow' ORDER BY step_slug $$,
22
+ array['step_one', 'step_three', 'step_two']::text [],
23
+ 'No duplicated steps were created'
24
+ );
25
+
26
+ -- Verify the state hasn't changed
27
+ select results_eq(
28
+ $$ SELECT DISTINCT flow_slug FROM pgflow.flows $$,
29
+ array['test_flow']::text [],
30
+ 'No duplicated flows were created'
31
+ );
32
+
33
+ select * from finish();
34
+ rollback;
package/tsconfig.json ADDED
@@ -0,0 +1,16 @@
1
+ {
2
+ "extends": "../../tsconfig.base.json",
3
+ "files": [],
4
+ "include": [],
5
+ "references": [
6
+ {
7
+ "path": "../dsl"
8
+ },
9
+ {
10
+ "path": "./tsconfig.lib.json"
11
+ },
12
+ {
13
+ "path": "./tsconfig.spec.json"
14
+ }
15
+ ]
16
+ }
@@ -0,0 +1,26 @@
1
+ {
2
+ "extends": "../../tsconfig.base.json",
3
+ "compilerOptions": {
4
+ "outDir": "dist",
5
+ "tsBuildInfoFile": "dist/tsconfig.lib.tsbuildinfo",
6
+ "emitDeclarationOnly": false,
7
+ "types": ["node", "vite/client"],
8
+ "allowImportingTsExtensions": true
9
+ },
10
+ "include": ["src/**/*.ts"],
11
+ "references": [{ "path": "../dsl/tsconfig.lib.json" }],
12
+ "exclude": [
13
+ "vite.config.ts",
14
+ "vite.config.mts",
15
+ "vitest.config.ts",
16
+ "vitest.config.mts",
17
+ "src/**/*.test.ts",
18
+ "src/**/*.spec.ts",
19
+ "src/**/*.test.tsx",
20
+ "src/**/*.spec.tsx",
21
+ "src/**/*.test.js",
22
+ "src/**/*.spec.js",
23
+ "src/**/*.test.jsx",
24
+ "src/**/*.spec.jsx"
25
+ ]
26
+ }
@@ -0,0 +1,35 @@
1
+ {
2
+ "extends": "../../tsconfig.base.json",
3
+ "compilerOptions": {
4
+ "outDir": "./out-tsc/vitest",
5
+ "types": [
6
+ "vitest/globals",
7
+ "vitest/importMeta",
8
+ "vite/client",
9
+ "node",
10
+ "vitest"
11
+ ],
12
+ "allowImportingTsExtensions": true
13
+ },
14
+ "include": [
15
+ "vite.config.ts",
16
+ "vite.config.mts",
17
+ "vitest.config.ts",
18
+ "vitest.config.mts",
19
+ "src/**/*.test.ts",
20
+ "src/**/*.spec.ts",
21
+ "src/**/*.test.tsx",
22
+ "src/**/*.spec.tsx",
23
+ "src/**/*.test.js",
24
+ "src/**/*.spec.js",
25
+ "src/**/*.test.jsx",
26
+ "src/**/*.spec.jsx",
27
+ "src/**/*.d.ts",
28
+ "__tests__/**/*.ts"
29
+ ],
30
+ "references": [
31
+ {
32
+ "path": "./tsconfig.lib.json"
33
+ }
34
+ ]
35
+ }
package/vite.config.ts ADDED
@@ -0,0 +1,57 @@
1
+ /// <reference types='vitest' />
2
+ import { defineConfig } from 'vite';
3
+ import dts from 'vite-plugin-dts';
4
+ import * as path from 'path';
5
+ import tsconfigPaths from 'vite-tsconfig-paths';
6
+
7
+ export default defineConfig({
8
+ root: __dirname,
9
+ cacheDir: '../../node_modules/.vite/pkgs/core',
10
+ plugins: [
11
+ tsconfigPaths(),
12
+ dts({
13
+ entryRoot: 'src',
14
+ tsconfigPath: path.join(__dirname, 'tsconfig.lib.json'),
15
+ }),
16
+ ],
17
+ // Uncomment this if you are using workers.
18
+ // worker: {
19
+ // plugins: [ nxViteTsPaths() ],
20
+ // },
21
+ // Configuration for building your library.
22
+ // See: https://vitejs.dev/guide/build.html#library-mode
23
+ build: {
24
+ outDir: './dist',
25
+ emptyOutDir: true,
26
+ reportCompressedSize: true,
27
+ commonjsOptions: {
28
+ transformMixedEsModules: true,
29
+ },
30
+ lib: {
31
+ // Could also be a dictionary or array of multiple entry points.
32
+ entry: 'src/index.ts',
33
+ name: 'core',
34
+ fileName: 'index',
35
+ // Change this to the formats you want to support.
36
+ // Don't forget to update your package.json as well.
37
+ formats: ['es'],
38
+ },
39
+ rollupOptions: {
40
+ // External packages that should not be bundled into your library.
41
+ external: [],
42
+ },
43
+ },
44
+ test: {
45
+ watch: false,
46
+ globals: true,
47
+ environment: 'node',
48
+ include: [
49
+ '{src,__tests__}/**/*.{test,spec,test-d,spec-d}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}',
50
+ ],
51
+ reporters: ['default'],
52
+ coverage: {
53
+ reportsDirectory: './test-output/vitest/coverage',
54
+ provider: 'v8',
55
+ },
56
+ },
57
+ });