@oomol-lab/open-flow 0.1.0-beta.77 → 0.1.0-beta.78

package/src/runtime/common/runtime.ts

@@ -1051,14 +1051,13 @@ export function createBase64() {
       const padding = text.endsWith('==') ? 2 : text.endsWith('=') ? 1 : 0
       const bytes = new Uint8Array((text.length / 4) * 3 - padding)
       for (let index = 0, offset = 0; index < text.length; index += 4) {
-        const digits = [0, 1, 2, 3].map((position) => {
+        let value = 0
+        for (let position = 0; position < 4; position++) {
           const code = text.charCodeAt(index + position)
-          if (code == 61 && index + position >= text.length - padding) return 0
-          const value = code < 128 ? values[code]! : 255
-          if (value == 255) throw new TypeError('Base64 content is invalid.')
-          return value
-        })
-        const value = (digits[0]! << 18) | (digits[1]! << 12) | (digits[2]! << 6) | digits[3]!
+          const digit = code == 61 && index + position >= text.length - padding ? 0 : code < 128 ? values[code]! : 255
+          if (digit == 255) throw new TypeError('Base64 content is invalid.')
+          value = (value << 6) | digit
+        }
         if (offset < bytes.length) bytes[offset++] = value >> 16
         if (offset < bytes.length) bytes[offset++] = (value >> 8) & 255
         if (offset < bytes.length) bytes[offset++] = value & 255

package/src/state/common/conformance.ts

@@ -1,5 +1,5 @@
 import type { JsonValue } from '../../flow/common/model.ts'
-import type { DedupeConfig } from './state.ts'
+import type { SkipDuplicatesConfig } from './state.ts'
 
 import { dequal } from 'dequal/lite'
 
@@ -7,31 +7,26 @@ export interface StateConformanceHarness {
   readonly maxIdentities: number
   /** Executes Code on the same initially empty Flow and scope on every call; returns its `result` output. */
   run(source: string): Promise<JsonValue>
-  /** Executes the same dedupe node across Runs, updating its business config and items. */
-  dedupe(config: DedupeConfig, items: JsonValue[]): Promise<{ items: JsonValue[]; duplicates: JsonValue[] }>
+  /** Executes the same Skip Duplicates node across Runs, updating its business config and items. */
+  skipDuplicates(config: SkipDuplicatesConfig, items: JsonValue[]): Promise<{ items: JsonValue[]; duplicates: JsonValue[] }>
 }
 
 export const stateConformanceCases: readonly { name: string; verify(harness: StateConformanceHarness): Promise<void> }[] = [
   {
-    name: 'persists watermarks by instant and resets changed modes atomically',
+    name: 'skips composite identities across successful Runs without depending on field order',
     async verify(harness) {
-      const config: DedupeConfig = { field: '', mode: 'increasing', space: 'node' }
-      const first = await harness.dedupe(config, ['2026-10-30T09:00:00+08:00', '2026-10-30T01:00:00Z', '2026-10-30T02:00:00Z'])
-      if (!dequal(first, { items: ['2026-10-30T09:00:00+08:00', '2026-10-30T02:00:00Z'], duplicates: ['2026-10-30T01:00:00Z'] }))
-        throw new Error('Watermarks must compare instants and preserve item order.')
-      let rejected = false
-      try {
-        await harness.dedupe(config, [1])
-      } catch {
-        rejected = true
-      }
-      if (!rejected) throw new Error('Mixed watermark types must fail.')
-      const second = await harness.dedupe(config, ['2026-10-30T02:00:00Z', '2026-10-30T03:00:00Z'])
-      if (!dequal(second, { items: ['2026-10-30T03:00:00Z'], duplicates: ['2026-10-30T02:00:00Z'] }))
-        throw new Error('A failed batch must preserve the previous watermark.')
-      await harness.dedupe({ ...config, mode: 'new' }, [])
-      const reset = await harness.dedupe(config, [5, 3, 5])
-      if (!dequal(reset, { items: [5, 3], duplicates: [5] })) throw new Error('A mode reset must discard previous watermarks, including on empty batches.')
+      const items = [
+        { company: 'a', id: 1 },
+        { company: 'b', id: 1 },
+        { company: 'a', id: 1 },
+      ]
+      const first = await harness.skipDuplicates({ fields: ['company', 'id'] }, items)
+      if (!dequal(first, { items: items.slice(0, 2), duplicates: [items[2]] }))
+        throw new Error('Composite identities must retain the first item in input order.')
+      const second = await harness.skipDuplicates({ fields: ['id', 'company'] }, items)
+      if (!dequal(second, { items: [], duplicates: items })) throw new Error('Successful Run history must persist independently of field order.')
+      const changed = await harness.skipDuplicates({ fields: ['id'] }, items)
+      if (!dequal(changed, { items: [items[0]], duplicates: items.slice(1) })) throw new Error('The field definition must participate in identity.')
     },
   },
   {

package/src/state/common/state.ts

@@ -16,21 +16,25 @@ export const stateClearSchema = z.strictObject({
 })
 export const dedupeClearSchema = z.strictObject({ scope: z.enum(['draft', 'live']), space: z.string().regex(/^(flow|node:.+)$/) })
 export type DedupeIdentity = string | number
-export const dedupeConfigSchema = z.strictObject({
-  field: z.string().default(''),
-  mode: z.enum(['new', 'increasing']).default('new'),
-  space: z.enum(['node', 'flow']).default('node'),
+export const skipDuplicatesConfigSchema = z.strictObject({
+  fields: z
+    .array(z.string().refine((field) => field.length > 0 && field.split('.').every((part) => part.length > 0), 'Use a nonempty dot-separated field path.'))
+    .refine((fields) => new Set(fields).size === fields.length, 'Identity fields must be distinct.')
+    .default([]),
 })
-export type DedupeConfig = z.infer<typeof dedupeConfigSchema>
-export interface DedupeExecutor extends DedupeConfig {
-  readonly kind: 'dedupe'
+export type SkipDuplicatesConfig = z.infer<typeof skipDuplicatesConfigSchema>
+export interface SkipDuplicatesExecutor extends SkipDuplicatesConfig {
+  readonly kind: 'skip-duplicates'
 }
-export const dedupeExecutorSchema = dedupeConfigSchema.extend({ kind: z.literal('dedupe') })
+export const skipDuplicatesExecutorSchema = skipDuplicatesConfigSchema.extend({ kind: z.literal('skip-duplicates') })
 
-export function dedupeTask(config: DedupeConfig = dedupeConfigSchema.parse({}), name = 'Remove Duplicates'): ManagedTaskDefinitionOf<DedupeExecutor> {
+export function skipDuplicatesTask(
+  config: SkipDuplicatesConfig = skipDuplicatesConfigSchema.parse({}),
+  name = 'Skip Duplicates',
+): ManagedTaskDefinitionOf<SkipDuplicatesExecutor> {
   return {
     name,
-    executor: { ...config, kind: 'dedupe' },
+    executor: { ...config, kind: 'skip-duplicates' },
     inputs: [{ handle: 'items', jsonSchema: { type: 'array' }, nullable: false }],
     outputs: ['items', 'duplicates'].map((handle) => ({ handle, jsonSchema: { type: 'array' }, nullable: false })),
   }
@@ -39,13 +43,13 @@ export function dedupeTask(config: DedupeConfig = dedupeConfigSchema.parse({}),
 export class StateError extends Error {
   readonly retryable = false
   readonly catchable: boolean
-  readonly code: 'capability.invalid' | 'capability.denied' | 'dedupe.full' | 'limit-exceeded'
+  readonly code: 'capability.invalid' | 'capability.denied' | 'dedupe.full' | 'limit-exceeded' | 'node.input-invalid'
 
   constructor(code: StateError['code'], message: string) {
     super(message)
     this.name = 'StateError'
     this.code = code
-    this.catchable = code === 'dedupe.full'
+    this.catchable = code === 'dedupe.full' || code === 'node.input-invalid'
   }
 }
 
@@ -67,63 +71,31 @@ export async function hashIdentities(ids: readonly DedupeIdentity[]): Promise<Ui
   )
 }
 
-export function watermarkValue(id: DedupeIdentity): number {
-  if (typeof id === 'number') {
-    if (!Number.isFinite(id)) throw new StateError('capability.invalid', 'Watermarks must be finite.')
-    return id
-  }
-  const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,3}))?(Z|[+-]\d{2}:\d{2})$/.exec(id)
-  if (match == null) throw new StateError('capability.invalid', 'Watermarks require RFC 3339 timestamps with an explicit timezone.')
-  const [, year, month, day, hour, minute, second, , zone] = match
-  const y = Number(year)
-  const days = [31, y % 4 === 0 && (y % 100 !== 0 || y % 400 === 0) ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][Number(month) - 1] ?? 0
-  const result = Date.parse(id)
-  if (
-    Number(month) < 1 ||
-    Number(month) > 12 ||
-    Number(day) < 1 ||
-    Number(day) > days ||
-    Number(hour) > 23 ||
-    Number(minute) > 59 ||
-    Number(second) > 59 ||
-    (zone !== 'Z' && (Number(zone!.slice(1, 3)) > 23 || Number(zone!.slice(4)) > 59)) ||
-    !Number.isFinite(result)
-  )
-    throw new StateError('capability.invalid', 'Watermarks require valid calendar timestamps.')
-  return result
-}
-
-export function increasingIdentities(ids: readonly DedupeIdentity[], watermark: DedupeIdentity | null) {
-  const type = typeof (watermark ?? ids[0])
-  if (ids.some((id) => typeof id !== type)) throw new StateError('capability.invalid', 'Watermark types must not be mixed.')
-  const initial = watermark == null ? -Infinity : watermarkValue(watermark)
-  const seen = new Set<number>()
-  let maximum = initial
-  let next = watermark
-  const inserted = ids.map((id) => {
-    const value = watermarkValue(id)
-    const fresh = value > initial && !seen.has(value)
-    seen.add(value)
-    if (value > maximum) {
-      maximum = value
-      next = id
-    }
-    return fresh
+/** Canonical identities include the field definition and preserve scalar types. */
+export function skipDuplicatesIdentities(items: JsonValue, fields: readonly string[]): string[] {
+  if (!Array.isArray(items)) throw new StateError('node.input-invalid', 'Skip Duplicates requires an array of items.')
+  const paths = fields.toSorted()
+  return items.map((item, index) => {
+    const values = (paths.length === 0 ? [''] : paths).map((field) => {
+      const value =
+        field === ''
+          ? item
+          : field
+              .split('.')
+              .reduce<JsonValue | undefined>(
+                (current, key) =>
+                  current != null && typeof current === 'object' && Object.hasOwn(current, key) ? (current as Record<string, JsonValue>)[key] : undefined,
+                item,
+              )
+      if (typeof value !== 'string' && !(typeof value === 'number' && Number.isFinite(value)))
+        throw new StateError(
+          'node.input-invalid',
+          `Item ${index + 1}, ${field === '' ? 'the item itself' : `field ${JSON.stringify(field)}`} must be a string or finite number.`,
+        )
+      return value
+    })
+    return JSON.stringify([paths, values])
   })
-  return { inserted, watermark: next }
-}
-
-export function itemIdentities(items: JsonValue, field: string): DedupeIdentity[] {
-  if (!Array.isArray(items)) throw new StateError('capability.invalid', 'Deduplication requires an array of items.')
-  const path = field === '' ? [] : field.split('.')
-  return dedupeIdentities(
-    items.map((item) =>
-      path.reduce<JsonValue | undefined>(
-        (value, key) => (value != null && typeof value === 'object' && Object.hasOwn(value, key) ? (value as Record<string, JsonValue>)[key] : undefined),
-        item,
-      ),
-    ),
-  )
 }
 
 export interface FlowStateView {
@@ -139,9 +111,9 @@ export interface FlowStateView {
   readonly values: readonly { readonly key: string; readonly size: number; readonly updatedAt: number; readonly value?: JsonValue }[]
   readonly spaces: readonly {
     readonly space: string
-    readonly mode: 'new' | 'increasing'
     readonly count: number
-    readonly watermark: DedupeIdentity | null
+    readonly pendingCount: number
+    readonly reservedCount: number
     readonly updatedAt: number
   }[]
 }
@@ -160,9 +132,9 @@ export const flowStateViewSchema = z.object({
   spaces: z.array(
     z.object({
       space: z.string(),
-      mode: z.enum(['new', 'increasing']),
       count: z.number().nonnegative(),
-      watermark: z.union([z.string(), z.number(), z.null()]),
+      pendingCount: z.number().nonnegative(),
+      reservedCount: z.number().nonnegative(),
       updatedAt: z.number(),
     }),
   ),