@effect-app/infra 4.0.0-beta.212 → 4.0.0-beta.214

package/src/Store/Memory.ts

@@ -2,8 +2,11 @@
 
 import { Array, Context, Effect, flow, type NonEmptyReadonlyArray, Option, Order, pipe, Ref, Result, Semaphore, Struct } from "effect-app"
 import { NonEmptyString255 } from "effect-app/Schema"
+import { assertUnreachable } from "effect-app/utils"
 import { InfraLogger } from "../logger.js"
+import type { FilterResult } from "../Model/filter/filterApi.js"
 import type { FieldValues } from "../Model/filter/types.js"
+import type { ComputedProjectionIrExpression } from "../Model/query.js"
 import { annotateDb } from "../otel.js"
 import { codeFilter, codeFilter3_ } from "./codeFilter.js"
 import { type FilterArgs, type PersistenceModelType, type Store, type StoreConfig, StoreMaker } from "./service.js"
@@ -14,6 +17,87 @@ export function get(obj: any, path: string): any {
   return path.split(".").reduce((res: any, key: string) => (res != null ? res[key] : res), obj)
 }
 
+const stripRelationFilterPaths = (state: readonly FilterResult[], relationPath: string): readonly FilterResult[] => {
+  const prefix = `${relationPath}.-1.`
+  return state.map((entry) =>
+    "path" in entry
+      ? {
+        ...entry,
+        path: entry.path.startsWith(prefix) ? entry.path.slice(prefix.length) : entry.path
+      }
+      : {
+        ...entry,
+        result: stripRelationFilterPaths(entry.result, relationPath)
+      }
+  )
+}
+
+const emptyValueFor = (tag: ComputedProjectionIrExpression["_tag"]) => {
+  switch (tag) {
+    case "relation-count":
+    case "relation-distinct-count":
+    case "relation-sum":
+      return 0
+    case "relation-any":
+      return false
+    case "relation-every":
+      return true
+    case "relation-collect":
+      return [] as unknown[]
+    default:
+      return assertUnreachable(tag)
+  }
+}
+
+const computeProjectionValue = (
+  row: FieldValues,
+  computed: ComputedProjectionIrExpression
+) => {
+  const relation = get(row, computed.path)
+  if (!Array.isArray(relation)) {
+    return emptyValueFor(computed._tag)
+  }
+  const filter = stripRelationFilterPaths(computed.filter, computed.path)
+  const matches = (value: unknown) => codeFilter3_(filter, value)
+  switch (computed._tag) {
+    case "relation-count":
+      return relation.reduce<number>((acc, value) => matches(value) ? acc + 1 : acc, 0)
+    case "relation-any":
+      return relation.some(matches)
+    case "relation-every":
+      return relation.every(matches)
+    case "relation-distinct-count": {
+      const seen = new Set<unknown>()
+      for (const value of relation) {
+        if (matches(value)) seen.add(get(value, computed.field))
+      }
+      return seen.size
+    }
+    case "relation-sum":
+      return relation.reduce<number>((acc, value) => {
+        if (!matches(value)) return acc
+        const v = get(value, computed.field)
+        return acc + (typeof v === "number" ? v : Number(v) || 0)
+      }, 0)
+    case "relation-collect": {
+      const out: unknown[] = []
+      const seen = computed.distinct ? new Set<unknown>() : undefined
+      for (const value of relation) {
+        if (!matches(value)) continue
+        const v = get(value, computed.field)
+        if (seen) {
+          if (seen.has(v)) continue
+          seen.add(v)
+        }
+        out.push(v)
+      }
+      return out
+    }
+    default:
+      return assertUnreachable(computed)
+  }
+}
+
 export function memFilter<T extends FieldValues, U extends keyof T = never>(f: FilterArgs<T, U>) {
   type M = U extends undefined ? T : Pick<T, U>
   return ((c: T[]): M[] => {
@@ -21,16 +105,24 @@ export function memFilter<T extends FieldValues, U extends keyof T = never>(f: F
       const sel = f.select
       if (!sel) return r as M[]
       return r.map((i) => {
-        const [keys, subKeys] = pipe(
+        const [keys, entries] = pipe(
           sel,
-          Array.partition((r) =>
-            typeof r === "string" ? Result.fail(String(r)) : Result.succeed(r as { key: string; subKeys: string[] })
-          )
+          Array.partition((entry) => typeof entry === "string" ? Result.fail(String(entry)) : Result.succeed(entry))
         )
+        const subKeys = entries.filter((entry): entry is { key: string; subKeys: readonly string[] } =>
+          typeof entry === "object" && entry !== null && "subKeys" in entry
+        )
+        const computedKeys = entries.filter((entry): entry is {
+          key: string
+          computed: ComputedProjectionIrExpression
+        } => typeof entry === "object" && entry !== null && "computed" in entry)
         const n = Struct.pick(i, keys)
         subKeys.forEach((subKey) => {
           n[subKey.key] = i[subKey.key]!.map(Struct.pick(subKey.subKeys as never[]))
         })
+        computedKeys.forEach((entry) => {
+          ;(n as Record<string, unknown>)[entry.key] = computeProjectionValue(i, entry.computed)
+        })
         return n as M
       })
     }

package/src/Store/SQL/Pg.ts

@@ -6,6 +6,7 @@ import { SqlClient } from "effect/unstable/sql"
 import { OptimisticConcurrencyException } from "../../errors.js"
 import { InfraLogger } from "../../logger.js"
 import type { FieldValues } from "../../Model/filter/types.js"
+import type { ComputedProjectionIrExpression } from "../../Model/query.js"
 import { annotateDb } from "../../otel.js"
 import { storeId } from "../Memory.js"
 import { type FilterArgs, type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, StoreMaker } from "../service.js"
@@ -217,7 +218,15 @@ const makePgStore = Effect.fnUntraced(function*({ prefix }: StorageConfig) {
                   tableName,
                   defaultValues,
                   f.select as
-                    | NonEmptyReadonlyArray<string | { key: string; subKeys: readonly string[] }>
+                    | NonEmptyReadonlyArray<
+                      string | {
+                        key: string
+                        subKeys: readonly string[]
+                      } | {
+                        key: string
+                        computed: ComputedProjectionIrExpression
+                      }
+                    >
                     | undefined,
                   f.order,
                   f.skip,

package/src/Store/SQL/query.ts

@@ -3,6 +3,7 @@ import { Effect, type NonEmptyReadonlyArray } from "effect-app"
 import { assertUnreachable } from "effect-app/utils"
 import { InfraLogger } from "../../logger.js"
 import type { FilterR, FilterResult } from "../../Model/filter/filterApi.js"
+import type { ComputedProjectionIrExpression } from "../../Model/query.js"
 import { isRelationCheck } from "../codeFilter.js"
 
 export interface SQLDialect {
@@ -153,7 +154,15 @@ export function buildWhereSQLQuery(
   filter: readonly FilterResult[],
   tableName: string,
   defaultValues: Record<string, unknown>,
-  select?: NonEmptyReadonlyArray<string | { key: string; subKeys: readonly string[] }>,
+  select?: NonEmptyReadonlyArray<
+    string | {
+      key: string
+      subKeys: readonly string[]
+    } | {
+      key: string
+      computed: ComputedProjectionIrExpression
+    }
+  >,
   order?: NonEmptyReadonlyArray<{ key: string; direction: "ASC" | "DESC" }>,
   skip?: number,
   limit?: number
@@ -378,6 +387,58 @@ export function buildWhereSQLQuery(
     return s
   }
 
+  const computedSelectExpr = (key: string, computed: ComputedProjectionIrExpression): string => {
+    const relationPath = dottedToJsonPath(computed.path)
+    const relationAlias = `_${computed.path}`
+    const relationFrom = dialect.jsonEachFrom(relationPath, relationAlias)
+    const whereClause = () =>
+      computed.filter.length > 0
+        ? ` WHERE ${print(computed.filter, computed.path, false)}`
+        : ""
+    const boolExpr = (sqlExpr: string) =>
+      dialect.jsonColumnType === "JSON"
+        ? `CASE WHEN ${sqlExpr} THEN 'true' ELSE 'false' END AS "${key}"`
+        : `${sqlExpr} AS "${key}"`
+    switch (computed._tag) {
+      case "relation-count":
+        return `(SELECT COUNT(1) FROM ${relationFrom}${whereClause()}) AS "${key}"`
+      case "relation-any":
+        return boolExpr(`EXISTS(SELECT 1 FROM ${relationFrom}${whereClause()})`)
+      case "relation-every":
+        // ∀x.P(x) ≡ ¬∃x.¬P(x). When no filter, no element exists that violates ⊤ → true.
+        return boolExpr(
+          computed.filter.length === 0
+            ? `1=1`
+            : `NOT EXISTS(SELECT 1 FROM ${relationFrom} WHERE NOT (${print(computed.filter, computed.path, false)}))`
+        )
+      case "relation-distinct-count": {
+        const fieldExtract = dialect.jsonExtractElement(relationAlias, computed.field)
+        return `(SELECT COUNT(DISTINCT ${fieldExtract}) FROM ${relationFrom}${whereClause()}) AS "${key}"`
+      }
+      case "relation-sum": {
+        const fieldExtract = dialect.jsonExtractElement(relationAlias, computed.field)
+        const cast = dialect.jsonColumnType === "JSON"
+          ? `CAST(${fieldExtract} AS REAL)`
+          : `(${fieldExtract})::numeric`
+        return `(SELECT COALESCE(SUM(${cast}), 0) FROM ${relationFrom}${whereClause()}) AS "${key}"`
+      }
+      case "relation-collect": {
+        const fieldExtract = dialect.jsonExtractElement(relationAlias, computed.field)
+        if (dialect.jsonColumnType === "JSON") {
+          // sqlite: json_group_array does not accept DISTINCT; emulate via inner DISTINCT subquery
+          if (computed.distinct) {
+            return `(SELECT COALESCE(json_group_array(__v), json_array()) FROM (SELECT DISTINCT ${fieldExtract} AS __v FROM ${relationFrom}${whereClause()})) AS "${key}"`
+          }
+          return `(SELECT COALESCE(json_group_array(${fieldExtract}), json_array()) FROM ${relationFrom}${whereClause()}) AS "${key}"`
+        }
+        const aggArg = computed.distinct ? `DISTINCT ${fieldExtract}` : fieldExtract
+        return `(SELECT COALESCE(jsonb_agg(${aggArg}), '[]'::jsonb) FROM ${relationFrom}${whereClause()}) AS "${key}"`
+      }
+      default:
+        return assertUnreachable(computed)
+    }
+  }
+
   const getSelectExpr = (): string => {
     if (!select) return "id, _etag, data"
     const fields = select.map((s) => {
@@ -386,6 +447,9 @@ export function buildWhereSQLQuery(
         if (s === "_etag") return `_etag`
         return `${dialect.jsonExtractJson(s)} AS "${s}"`
       }
+      if ("computed" in s) {
+        return computedSelectExpr(s.key, s.computed)
+      }
       return `${dialect.jsonExtractJson(s.key)} AS "${s.key}"`
     })
     return fields.join(", ")

package/src/Store/SQL.ts

@@ -7,6 +7,7 @@ import { SqlClient } from "effect/unstable/sql"
 import { OptimisticConcurrencyException } from "../errors.js"
 import { InfraLogger } from "../logger.js"
 import type { FieldValues } from "../Model/filter/types.js"
+import type { ComputedProjectionIrExpression } from "../Model/query.js"
 import { annotateDb, type DbSystem } from "../otel.js"
 import { storeId } from "./Memory.js"
 import { type FilterArgs, type PersistenceModelType, type StorageConfig, type Store, type StoreConfig, StoreMaker } from "./service.js"
@@ -236,7 +237,15 @@ function makeSQLStoreInt(system: DbSystem, dialect: SQLDialect, jsonColumnType:
                         defaultValues,
                         f
                           .select as
-                            | NonEmptyReadonlyArray<string | { key: string; subKeys: readonly string[] }>
+                            | NonEmptyReadonlyArray<
+                              string | {
+                                key: string
+                                subKeys: readonly string[]
+                              } | {
+                                key: string
+                                computed: ComputedProjectionIrExpression
+                              }
+                            >
                             | undefined,
                         f
                           .order,
@@ -580,7 +589,15 @@ function makeSQLiteStorePerNs(
                       defaultValues,
                       f
                         .select as
-                          | NonEmptyReadonlyArray<string | { key: string; subKeys: readonly string[] }>
+                          | NonEmptyReadonlyArray<
+                            string | {
+                              key: string
+                              subKeys: readonly string[]
+                            } | {
+                              key: string
+                              computed: ComputedProjectionIrExpression
+                            }
+                          >
                           | undefined,
                       f
                         .order as NonEmptyReadonlyArray<{ key: string; direction: "ASC" | "DESC" }> | undefined,

package/src/Store/service.ts

@@ -6,7 +6,7 @@ import type { OptimisticConcurrencyException } from "../errors.js"
 import type { FilterResult } from "../Model/filter/filterApi.js"
 import type { FieldValues } from "../Model/filter/types.js"
 import type { FieldPath } from "../Model/filter/types/path/index.js"
-import { type RawQuery } from "../Model/query.js"
+import type { ComputedProjectionIrExpression, RawQuery } from "../Model/query.js"
 
 export interface StoreConfig<E> {
   partitionValue: (e?: E) => string
@@ -61,7 +61,14 @@ export interface O<TFieldValues extends FieldValues> {
 export interface FilterArgs<Encoded extends FieldValues, U extends keyof Encoded = never> {
   t: Encoded
   filter?: Filter | undefined
-  select?: NonEmptyReadonlyArray<U | { key: string; subKeys: readonly string[] }> | undefined
+  select?:
+    | NonEmptyReadonlyArray<
+      U | { key: string; subKeys: readonly string[] } | {
+        key: string
+        computed: ComputedProjectionIrExpression
+      }
+    >
+    | undefined
   order?: NonEmptyReadonlyArray<O<NoInfer<Encoded>>>
   limit?: number | undefined
   skip?: number | undefined

package/test/query.test.ts

@@ -6,7 +6,7 @@ import { Context, Effect, flow, Layer, Option, pipe, S, Struct } from "effect-ap
 import { inspect } from "util"
 import { expect, expectTypeOf, it } from "vitest"
 import { setupRequestContextFromCurrent } from "../src/api/setupRequest.js"
-import { and, count, make, one, or, order, page, project, type QueryEnd, type QueryProjection, type QueryWhere, toFilter, where } from "../src/Model/query.js"
+import { and, computed, count, make, one, or, order, page, project, projectComputed, type QueryEnd, type QueryProjection, type QueryWhere, relation, toFilter, where } from "../src/Model/query.js"
 import { makeRepo } from "../src/Model/Repository.js"
 import { RepositoryRegistryLive } from "../src/Model/Repository/Registry.js"
 import { memFilter, MemoryStoreLive } from "../src/Store/Memory.js"
@@ -575,6 +575,161 @@ it(
       .pipe(Effect.provide(TestStoreLive), setupRequestContextFromCurrent(), Effect.runPromise)
 )
 
+it("projectComputed sets computed IR and forces project mode", () => {
+  const baseSchema = S.Struct({
+    id: S.String,
+    items: S.Array(S.Struct({
+      state: S.Struct({
+        _tag: S.String
+      })
+    }))
+  })
+  const query = make<S.Codec.Encoded<typeof baseSchema>>().pipe(
+    projectComputed(
+      S.Struct({
+        pickedCount: S.NonNegativeInt
+      }),
+      computed({
+        pickedCount: relation<S.Codec.Encoded<typeof baseSchema>>("items").count(where("state._tag", "Picked"))
+      })
+    )
+  )
+  const interpreted = toFilter(query, baseSchema)
+  expect(interpreted.mode).toBe("project")
+  expect(interpreted.select).toEqual([
+    {
+      key: "pickedCount",
+      computed: {
+        _tag: "relation-count",
+        path: "items",
+        filter: [{ t: "where", path: "items.-1.state._tag", op: "eq", value: "Picked" }]
+      }
+    }
+  ])
+  expect(interpreted.computed?.["pickedCount"]?._tag).toBe("relation-count")
+  expect(interpreted.computed?.["pickedCount"]?.path).toBe("items")
+  expect(interpreted.computed?.["pickedCount"]?.filter).toEqual([
+    { t: "where", path: "items.-1.state._tag", op: "eq", value: "Picked" }
+  ])
+})
+
+it("projectComputed validates extra computed keys", () => {
+  const baseSchema = S.Struct({
+    id: S.String,
+    items: S.Array(S.Struct({ value: S.Number }))
+  })
+  const query = make<S.Codec.Encoded<typeof baseSchema>>().pipe(
+    projectComputed(
+      S.Struct({ id: S.String }),
+      computed({
+        pickedCount: relation<S.Codec.Encoded<typeof baseSchema>>("items").count()
+      })
+    )
+  )
+  expect(() => toFilter(query, baseSchema)).toThrowError("Computed projection keys must exist in projection schema")
+})
+
+it("projection schema with computed fields fails without computed map", () => {
+  const baseSchema = S.Struct({
+    id: S.String,
+    items: S.Array(S.Struct({ value: S.Number }))
+  })
+  const query = make<S.Codec.Encoded<typeof baseSchema>>().pipe(
+    projectComputed(S.Struct({ pickedCount: S.NonNegativeInt }), computed({}))
+  )
+  expect(() => toFilter(query, baseSchema)).toThrowError("Missing computed projections for schema keys")
+})
+
+it("projectComputed.every emits relation-every IR", () => {
+  const baseSchema = S.Struct({
+    id: S.String,
+    items: S.Array(S.Struct({ state: S.Struct({ _tag: S.String }) }))
+  })
+  const query = make<S.Codec.Encoded<typeof baseSchema>>().pipe(
+    projectComputed(
+      S.Struct({ allPicked: S.Boolean }),
+      computed({
+        allPicked: relation<S.Codec.Encoded<typeof baseSchema>>("items").every(where("state._tag", "Picked"))
+      })
+    )
+  )
+  const interpreted = toFilter(query, baseSchema)
+  expect(interpreted.computed?.["allPicked"]?._tag).toBe("relation-every")
+  expect(interpreted.computed?.["allPicked"]?.path).toBe("items")
+  expect(interpreted.computed?.["allPicked"]?.filter).toEqual([
+    { t: "where", path: "items.-1.state._tag", op: "eq", value: "Picked" }
+  ])
+})
+
+it("projectComputed.distinctCount emits relation-distinct-count IR with field", () => {
+  const baseSchema = S.Struct({
+    id: S.String,
+    items: S.Array(S.Struct({ rowId: S.String, state: S.Struct({ _tag: S.String }) }))
+  })
+  const query = make<S.Codec.Encoded<typeof baseSchema>>().pipe(
+    projectComputed(
+      S.Struct({ positionCount: S.NonNegativeInt }),
+      computed({
+        positionCount: relation<S.Codec.Encoded<typeof baseSchema>>("items").distinctCount(
+          "rowId",
+          where("state._tag", "neq", "cancelled")
+        )
+      })
+    )
+  )
+  const interpreted = toFilter(query, baseSchema)
+  const ir = interpreted.computed?.["positionCount"]
+  expect(ir?._tag).toBe("relation-distinct-count")
+  expect((ir as { field: string } | undefined)?.field).toBe("rowId")
+  expect(ir?.filter).toEqual([
+    { t: "where", path: "items.-1.state._tag", op: "neq", value: "cancelled" }
+  ])
+})
+
+it("projectComputed.sum emits relation-sum IR with field", () => {
+  const baseSchema = S.Struct({
+    id: S.String,
+    items: S.Array(S.Struct({ weight: S.Number }))
+  })
+  const query = make<S.Codec.Encoded<typeof baseSchema>>().pipe(
+    projectComputed(
+      S.Struct({ totalWeight: S.Number }),
+      computed({ totalWeight: relation<S.Codec.Encoded<typeof baseSchema>>("items").sum("weight") })
+    )
+  )
+  const interpreted = toFilter(query, baseSchema)
+  const ir = interpreted.computed?.["totalWeight"]
+  expect(ir?._tag).toBe("relation-sum")
+  expect((ir as { field: string } | undefined)?.field).toBe("weight")
+  expect(ir?.filter).toEqual([])
+})
+
+it("projectComputed.collect / collectDistinct emit relation-collect IR", () => {
+  const baseSchema = S.Struct({
+    id: S.String,
+    items: S.Array(S.Struct({ articleId: S.String }))
+  })
+  const query = make<S.Codec.Encoded<typeof baseSchema>>().pipe(
+    projectComputed(
+      S.Struct({
+        all: S.Array(S.String),
+        distinct: S.Array(S.String)
+      }),
+      computed({
+        all: relation<S.Codec.Encoded<typeof baseSchema>>("items").collect("articleId"),
+        distinct: relation<S.Codec.Encoded<typeof baseSchema>>("items").collectDistinct("articleId")
+      })
+    )
+  )
+  const interpreted = toFilter(query, baseSchema)
+  const all = interpreted.computed?.["all"]
+  const distinct = interpreted.computed?.["distinct"]
+  expect(all?._tag).toBe("relation-collect")
+  expect((all as { distinct: boolean } | undefined)?.distinct).toBe(false)
+  expect(distinct?._tag).toBe("relation-collect")
+  expect((distinct as { distinct: boolean } | undefined)?.distinct).toBe(true)
+})
+
 it(
   "doesn't mess when refining fields",
   () =>

package/test/rawQuery.test.ts

@@ -2,7 +2,7 @@ import { describe, expect, it } from "@effect/vitest"
 import { Array, Config, Context, Effect, flow, Layer, ManagedRuntime, Redacted, References, Result, S } from "effect-app"
 import { LogLevels } from "effect-app/utils"
 import { setupRequestContextFromCurrent } from "../src/api/setupRequest.js"
-import { and, or, project, where, whereEvery, whereSome } from "../src/Model/query.js"
+import { and, computed, or, project, projectComputed, relation, where, whereEvery, whereSome } from "../src/Model/query.js"
 import { makeRepo } from "../src/Model/Repository/makeRepo.js"
 import { RepositoryRegistryLive } from "../src/Model/Repository/Registry.js"
 import { CosmosStoreLayer } from "../src/Store/Cosmos.js"
@@ -356,6 +356,41 @@ describe("multi-level", () => {
       .pipe(Effect.provide(SomethingRepo.Test), rt.runPromise))
 })
 
+describe("computed projections", () => {
+  const test = Effect
+    .gen(function*() {
+      const repo = yield* SomethingRepo
+      const output = S.Struct({
+        id: S.String,
+        pickedCount: S.NonNegativeInt,
+        hasPicked: S.Boolean
+      })
+      const pickedFilter = where("value", "gt", 20)
+      const items = yield* repo.query(
+        projectComputed(
+          output,
+          computed({
+            pickedCount: relation<S.Codec.Encoded<typeof Something>>("items").count(pickedFilter),
+            hasPicked: relation<S.Codec.Encoded<typeof Something>>("items").any(pickedFilter)
+          })
+        )
+      )
+      expect(items).toStrictEqual([
+        { id: "1", pickedCount: 0, hasPicked: false },
+        { id: "2", pickedCount: 2, hasPicked: true }
+      ])
+    })
+    .pipe(setupRequestContextFromCurrent())
+
+  it.skipIf(!process.env["STORAGE_URL"])("works well in CosmosDB", () =>
+    test
+      .pipe(Effect.provide(SomethingRepo.TestCosmos), rt.runPromise))
+
+  it("works well in Memory", () =>
+    test
+      .pipe(Effect.provide(SomethingRepo.Test), rt.runPromise))
+})
+
 // FUTURE: we need something like this instead:
 /*
   const subQuery = <T extends FieldValues>() => <TKey extends keyof T>(key: TKey, type: "some" | "every" = "some") => make<T[TKey][number]>() // todo: mark that this is sub query on field "items"