@effect-app/infra 2.61.8 → 2.63.0

package/src/Store/Cosmos/query.ts

@@ -3,7 +3,7 @@
 import { Array, Effect, type NonEmptyReadonlyArray } from "effect-app"
 import { assertUnreachable } from "effect-app/utils"
 import { InfraLogger } from "../../logger.js"
-import type { FilterR, FilterResult } from "../../Model/filter/filterApi.js"
+import type { FilterR, FilterResult, Ops } from "../../Model/filter/filterApi.js"
 import { isRelationCheck } from "../codeFilter.js"
 import type { SupportedValues } from "../service.js"
 
@@ -125,7 +125,48 @@ export function buildWhereCosmosQuery3(
 
   let i = 0
 
-  const print = (state: readonly FilterResult[], values: any[], isRelation: string | null = null) => {
+  const flipOps = {
+    gt: "lt",
+    lt: "gt",
+    gte: "lte",
+    lte: "gte",
+    contains: "notContains",
+    notContains: "contains",
+    startsWith: "notStartsWith",
+    notStartsWith: "startsWith",
+    endsWith: "notEndsWith",
+    notEndsWith: "endsWith",
+    eq: "neq",
+    neq: "eq",
+    includes: "notIncludes",
+    notIncludes: "includes",
+    "includes-any": "notIncludes-any",
+    "notIncludes-any": "includes-any",
+    "includes-all": "notIncludes-all",
+    "notIncludes-all": "includes-all",
+    in: "notIn",
+    notIn: "in"
+  } satisfies Record<Ops, Ops>
+
+  const flippies = {
+    and: "or",
+    or: "and"
+  } satisfies Record<"and" | "or", "and" | "or">
+
+  const flip = (every: boolean) => (_: FilterResult): FilterResult =>
+    every
+      ? _.t === "where" || _.t === "or" || _.t === "and"
+        ? {
+          ..._,
+          t: _.t === "where"
+            ? _.t
+            : flippies[_.t],
+          op: flipOps[_.op]
+        }
+        : _
+      : _
+
+  const print = (state: readonly FilterResult[], values: any[], isRelation: string | null, every: boolean) => {
     let s = ""
     let l = 0
     const printN = (n: number) => {
@@ -144,48 +185,62 @@ export function buildWhereCosmosQuery3(
           break
         case "or-scope": {
           ++l
+          if (!every) every = e.relation === "every"
           const rel = isRelationCheck(e.result, isRelation)
           if (rel) {
             const rel = (e.result[0]! as { path: string }).path.split(".-1.")[0]
             s += isRelation
-              ? ` OR (\n${printN(l + 1)}${print(e.result, values)}\n${printN(l)})`
-              : ` OR (\n${printN(l + 1)}EXISTS(SELECT VALUE ${rel} FROM ${rel} IN f.${rel} WHERE ${
-                print(e.result, values, rel)
+              ? ` OR (\n${printN(l + 1)}${print(e.result, values, rel, every)}\n${printN(l)})`
+              : ` OR (\n${printN(l + 1)}${
+                every ? "NOT " : ""
+              }EXISTS(SELECT VALUE ${rel} FROM ${rel} IN f.${rel} WHERE ${
+                print(
+                  e
+                    .result
+                    .map(flip(every)),
+                  values,
+                  rel,
+                  every
+                )
               }))`
           } else {
-            s += ` OR (\n${printN(l + 1)}${print(e.result, values)}\n${printN(l)})`
+            s += ` OR (\n${printN(l + 1)}${print(e.result, values, null, every)}\n${printN(l)})`
           }
           --l
           break
         }
         case "and-scope": {
           ++l
+          if (!every) every = e.relation === "every"
           const rel = isRelationCheck(e.result, isRelation)
           if (rel) {
             const rel = (e.result[0]! as { path: string }).path.split(".-1.")[0]
             s += isRelation
-              ? ` AND (\n${printN(l + 1)}${print(e.result, values)}\n${printN(l)})`
-              : ` AND (\n${printN(l + 1)}EXISTS(SELECT VALUE ${rel} FROM ${rel} IN f.${rel} WHERE ${
-                print(e.result, values, rel)
+              ? ` AND (\n${printN(l + 1)}${print(e.result, values, rel, every)}\n${printN(l)})`
+              : ` AND (\n${printN(l + 1)}${
+                every ? "NOT " : ""
+              }EXISTS(SELECT VALUE ${rel} FROM ${rel} IN f.${rel} WHERE ${
+                print(e.result.map(flip(every)), values, rel, every)
               }))`
           } else {
-            s += ` AND (\n${printN(l + 1)}${print(e.result, values)}\n${printN(l)})`
+            s += ` AND (\n${printN(l + 1)}${print(e.result, values, null, every)}\n${printN(l)})`
           }
           --l
           break
         }
         case "where-scope": {
           // ;++l
+          if (!every) every = e.relation === "every"
           const rel = isRelationCheck(e.result, isRelation)
           if (rel) {
             const rel = (e.result[0]! as { path: string }).path.split(".-1.")[0]
             s += isRelation
-              ? `(\n${printN(l + 1)}${print(e.result, values)}\n${printN(l)})`
-              : `(\n${printN(l + 1)}EXISTS(SELECT VALUE ${rel} FROM ${rel} IN f.${rel} WHERE ${
-                print(e.result, values, rel)
+              ? `(\n${printN(l + 1)}${print(e.result, values, rel, every)}\n${printN(l)})`
+              : `(\n${printN(l + 1)}${every ? "NOT " : ""}EXISTS(SELECT VALUE ${rel} FROM ${rel} IN f.${rel} WHERE ${
+                print(e.result.map(flip(every)), values, rel, every)
               }))`
           } else {
-            s += `(\n${printN(l + 1)}${print(e.result, values)}\n${printN(l)})`
+            s += `(\n${printN(l + 1)}${print(e.result, values, null, every)}\n${printN(l)})`
           }
           // ;--l
           break
@@ -239,7 +294,7 @@ export function buildWhereCosmosQuery3(
     }
     FROM ${name} f
 
-    WHERE f.id != @id ${filter.length ? `AND (${print(filter, values.map((_) => _.value))})` : ""}
+    WHERE f.id != @id ${filter.length ? `AND (${print(filter, values.map((_) => _.value), null, false)})` : ""}
     ${order ? `ORDER BY ${order.map((_) => `${dottedToAccess(`f.${_.key}`)} ${_.direction}`).join(", ")}` : ""}
     ${skip !== undefined || limit !== undefined ? `OFFSET ${skip ?? 0} LIMIT ${limit ?? 999999}` : ""}`,
     parameters: [

package/src/Store/Cosmos.ts

@@ -54,6 +54,8 @@ function makeCosmosStore({ prefix }: StorageConfig) {
           const container = db.container(containerId)
           const bulk = container.items.bulk.bind(container.items)
           const execBatch = container.items.batch.bind(container.items)
+          // TODO: move the marker to a separate container and get rid of the checks on every query
+          // then need to clean up the actual data.. perhaps first do with a config toggle to prescribe to it.
           const importedMarkerId = containerId
 
           const bulkSet = (items: NonEmptyReadonlyArray<PM>) =>
@@ -288,7 +290,7 @@ function makeCosmosStore({ prefix }: StorageConfig) {
                 .sync(() =>
                   buildWhereCosmosQuery3(
                     idKey,
-                    filter ? [{ t: "where-scope", result: filter }] : [],
+                    filter ? [{ t: "where-scope", result: filter, relation: "some" }] : [],
                     name,
                     importedMarkerId,
                     defaultValues,

package/src/Store/codeFilter.ts

@@ -93,7 +93,8 @@ const codeFilter3__ = <E>(
   state: readonly FilterResult[],
   sut: E,
   statements: any[],
-  isRelation: string | null = null
+  isRelation: string | null,
+  every: boolean
 ): string => {
   let s = ""
   let l = 0
@@ -126,32 +127,34 @@ const codeFilter3__ = <E>(
         break
       case "or-scope": {
         ++l
+        if (!every) every = e.relation === "every"
         const rel = isRelationCheck(e.result, isRelation)
         if (rel) {
           const rel = (e.result[0]! as { path: string }).path.split(".-1.")[0]
           s += isRelation
-            ? ` || (\n${printN(l + 1)}${codeFilter3__(e.result, sut, statements, rel)}\n${printN(l)})`
-            : ` || (\n${printN(l + 1)}sut.${rel}.some(el => ${codeFilter3__(e.result, sut, statements, rel)})\n${
-              printN(l)
-            })`
+            ? ` || (\n${printN(l + 1)}${codeFilter3__(e.result, sut, statements, rel, every)}\n${printN(l)})`
+            : ` || (\n${printN(l + 1)}sut.${rel}.${every ? "every" : "some"}(el => ${
+              codeFilter3__(e.result, sut, statements, rel, every)
+            })\n${printN(l)})`
         } else {
-          s += ` || (\n${printN(l + 1)}${codeFilter3__(e.result, sut, statements)}\n${printN(l)})`
+          s += ` || (\n${printN(l + 1)}${codeFilter3__(e.result, sut, statements, null, every)}\n${printN(l)})`
         }
         --l
         break
       }
       case "and-scope": {
         ++l
+        if (!every) every = e.relation === "every"
         const rel = isRelationCheck(e.result, isRelation)
         if (rel) {
           const rel = (e.result[0]! as { path: string }).path.split(".-1.")[0]
           s += isRelation
-            ? ` && (\n${printN(l + 1)}${codeFilter3__(e.result, sut, statements, rel)}\n${printN(l)})`
-            : ` && (\n${printN(l + 1)}sut.${rel}.some(el => ${codeFilter3__(e.result, sut, statements, rel)})\n${
-              printN(l)
-            })`
+            ? ` && (\n${printN(l + 1)}${codeFilter3__(e.result, sut, statements, rel, every)}\n${printN(l)})`
+            : ` && (\n${printN(l + 1)}sut.${rel}.${every ? "every" : "some"}(el => ${
+              codeFilter3__(e.result, sut, statements, rel, every)
+            })\n${printN(l)})`
         } else {
-          s += ` && (\n${printN(l + 1)}${codeFilter3__(e.result, sut, statements)}\n${printN(l)})`
+          s += ` && (\n${printN(l + 1)}${codeFilter3__(e.result, sut, statements, null, every)}\n${printN(l)})`
         }
         --l
 
@@ -159,16 +162,17 @@ const codeFilter3__ = <E>(
       }
       case "where-scope": {
         // ;++l
+        if (!every) every = e.relation === "every"
         const rel = isRelationCheck(e.result, isRelation)
         if (rel) {
           const rel = (e.result[0]! as { path: string }).path.split(".-1.")[0]
           s += isRelation
-            ? `(\n${printN(l + 1)}${codeFilter3__(e.result, sut, statements, rel)}\n${printN(l)})`
-            : `(\n${printN(l + 1)}sut.${rel}.some(el => ${codeFilter3__(e.result, sut, statements, rel)})\n${
-              printN(l)
-            })`
+            ? `(\n${printN(l + 1)}${codeFilter3__(e.result, sut, statements, rel, every)}\n${printN(l)})`
+            : `(\n${printN(l + 1)}sut.${rel}.${every ? "every" : "some"}(el => ${
+              codeFilter3__(e.result, sut, statements, rel, every)
+            })\n${printN(l)})`
         } else {
-          s += `(\n${printN(l + 1)}${codeFilter3__(e.result, sut, statements)}\n${printN(l)})`
+          s += `(\n${printN(l + 1)}${codeFilter3__(e.result, sut, statements, null, every)}\n${printN(l)})`
         }
         // ;--l
         break
@@ -181,7 +185,7 @@ const codeFilter3__ = <E>(
 export const codeFilter3_ = <E>(state: readonly FilterResult[], sut: E): boolean => {
   const statements: any[] = [] // must be defined here to be used by eval.
   // always put everything inside a root scope.
-  const s = codeFilter3__([{ t: "where-scope", result: state }], sut, statements)
+  const s = codeFilter3__([{ t: "where-scope", result: state, relation: "some" }], sut, statements, null, false)
   return eval(s)
 }
 

package/test/dist/rawQuery.test.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rawQuery.test.d.ts","sourceRoot":"","sources":["../rawQuery.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAkD,cAAc,EAAuB,MAAM,YAAY,CAAA;AAChH,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAO5C,eAAO,MAAM,EAAE,iDAUb,CAAA"}
+{"version":3,"file":"rawQuery.test.d.ts","sourceRoot":"","sources":["../rawQuery.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAwD,cAAc,EAAuB,MAAM,YAAY,CAAA;AACtH,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAO5C,eAAO,MAAM,EAAE,iDAUb,CAAA"}

package/test/rawQuery.test.ts

@@ -1,8 +1,8 @@
 import { describe, expect, it } from "@effect/vitest"
-import { Array, Config, Effect, Layer, Logger, LogLevel, ManagedRuntime, Option, Redacted, S } from "effect-app"
+import { Array, Config, Effect, flow, Layer, Logger, LogLevel, ManagedRuntime, Option, Redacted, S } from "effect-app"
 import { LogLevels } from "effect-app/utils"
 import { setupRequestContextFromCurrent } from "../src/api/setupRequest.js"
-import { and, or, project, where } from "../src/Model/query.js"
+import { and, or, project, where, whereEvery, whereSome } from "../src/Model/query.js"
 import { makeRepo } from "../src/Model/Repository/makeRepo.js"
 import { CosmosStoreLayer } from "../src/Store/Cosmos.js"
 import { MemoryStoreLive } from "../src/Store/Memory.js"
@@ -146,6 +146,35 @@ describe("select first-level array fields", () => {
       .pipe(Effect.provide(SomethingRepo.Test), rt.runPromise))
 })
 
+const projected = S.Struct({ name: S.String, items: S.Array(S.Struct({ id: S.String, value: S.Number })) })
+
+const expected = [
+  {
+    name: "Item 2",
+    items: [
+      { id: "2-1", value: 30 },
+      { id: "2-2", value: 40 }
+    ]
+  }
+]
+
+const both = [
+  {
+    name: "Item 1",
+    items: [
+      { id: "1-1", value: 10 },
+      { id: "1-2", value: 20 }
+    ]
+  },
+  {
+    name: "Item 2",
+    items: [
+      { id: "2-1", value: 30 },
+      { id: "2-2", value: 40 }
+    ]
+  }
+]
+
 // NOTE: right now we cannot specify if all/"every" items must match the filter, or if at least one item (any/"some") must match the filter.
 // the current implementation is any/some, so we can always filter down in the code to narrow further..
 describe("filter first-level array fields as groups", () => {
@@ -153,8 +182,6 @@ describe("filter first-level array fields as groups", () => {
     .gen(function*() {
       const repo = yield* SomethingRepo
 
-      const projected = S.Struct({ name: S.String, items: S.Array(S.Struct({ id: S.String, value: S.Number })) })
-
       // ok crazy lol, "value" is a reserved word in CosmosDB, so we have to use t["value"] as a field name instead of t.value
       // deprecated; joins should be avoided because they're very expensive, and require DISTINCT to avoid duplicates
       // which might affect results in unexpected ways?
@@ -200,73 +227,104 @@ describe("filter first-level array fields as groups", () => {
         )
       })
 
+      expect(items).toStrictEqual(expected)
+      expect(itemsExists).toStrictEqual(expected)
+    })
+    .pipe(setupRequestContextFromCurrent())
+
+  it("works well in CosmosDB", () =>
+    test
+      .pipe(Effect.provide(SomethingRepo.TestCosmos), rt.runPromise))
+
+  it("works well in Memory", () =>
+    test
+      .pipe(Effect.provide(SomethingRepo.Test), rt.runPromise))
+})
+
+describe("1", () => {
+  const test = Effect
+    .gen(function*() {
+      const repo = yield* SomethingRepo
       const items2 = yield* repo.query(
-        where("items.-1.value", "gt", 20),
-        and("items.-1.description", "contains", "d item"),
+        whereSome(
+          "items",
+          flow(
+            where("value", "gt", 20),
+            and("description", "contains", "d item")
+          )
+        ),
         project(projected)
       )
+      expect(items2).toStrictEqual(expected)
 
       const items2Or = yield* repo.query(
-        where("items.-1.value", "gt", 20),
-        or("items.-1.description", "contains", "d item"),
+        whereSome(
+          "items",
+          flow(
+            where("value", "gt", 20),
+            or("description", "contains", "d item")
+          )
+        ),
         project(projected)
       )
 
+      expect(items2Or).toStrictEqual(both)
       // mixing relation check with scoped relationcheck
       const items3 = yield* repo.query(
-        where("items.-1.value", "gt", 20),
-        and(where("items.-1.description", "contains", "d item")),
+        whereSome(
+          "items",
+          flow(where("value", "gt", 20), and(where("description", "contains", "d item")))
+        ),
         project(projected)
       )
 
+      expect(items3).toStrictEqual(expected)
       const items3Or = yield* repo.query(
-        where("items.-1.value", "gt", 20),
-        or(where("items.-1.description", "contains", "d item")),
+        whereSome(
+          "items",
+          flow(
+            where("value", "gt", 20),
+            or(where("description", "contains", "d item"))
+          )
+        ),
         project(projected)
       )
 
-      // broken in cosmos db somehow... returns twice record 2??
-      // need to use DISTINCT..
-      // https://stackoverflow.com/questions/51855660/cosmos-db-joins-give-duplicate-results
+      expect(items3Or).toStrictEqual(both)
       const items4 = yield* repo.query(
-        where("items.-1.value", "gt", 10),
+        whereSome("items", where("value", "gt", 10)),
         project(projected)
       )
 
-      const expected = [
-        {
-          name: "Item 2",
-          items: [
-            { id: "2-1", value: 30 },
-            { id: "2-2", value: 40 }
-          ]
-        }
-      ]
+      expect(items4).toStrictEqual(both)
+    })
+    .pipe(setupRequestContextFromCurrent())
 
-      const both = [
-        {
-          name: "Item 1",
-          items: [
-            { id: "1-1", value: 10 },
-            { id: "1-2", value: 20 }
-          ]
-        },
-        {
-          name: "Item 2",
-          items: [
-            { id: "2-1", value: 30 },
-            { id: "2-2", value: 40 }
-          ]
-        }
-      ]
+  it("works well in CosmosDB", () =>
+    test
+      .pipe(Effect.provide(SomethingRepo.TestCosmos), rt.runPromise))
 
-      expect(items).toStrictEqual(expected)
-      expect(itemsExists).toStrictEqual(expected)
-      expect(items2).toStrictEqual(expected)
-      expect(items2Or).toStrictEqual(both)
-      expect(items3).toStrictEqual(expected)
-      expect(items3Or).toStrictEqual(both)
-      expect(items4).toStrictEqual(both)
+  it("works well in Memory", () =>
+    test
+      .pipe(Effect.provide(SomethingRepo.Test), rt.runPromise))
+})
+
+describe("multi-level", () => {
+  const test = Effect
+    .gen(function*() {
+      const repo = yield* SomethingRepo
+      const itemsCheckWithEvery = yield* repo.query(
+        whereEvery(
+          "items",
+          flow(
+            where("value", "gt", 20),
+            and("description", "contains", "d item")
+          )
+        ),
+        project(projected)
+      )
+
+      expect(itemsCheckWithEvery).toStrictEqual([])
     })
     .pipe(setupRequestContextFromCurrent())