@effect-app/infra 0.117.0 → 0.119.0

package/_src/services/Store/Cosmos.ts

@@ -3,21 +3,30 @@
 import { CosmosClient } from "@effect-app/infra-adapters/cosmos-client"
 import { omit } from "@effect-app/prelude/utils"
 import { OptimisticConcurrencyException } from "../../errors.js"
+import {
+  buildCosmosQuery,
+  buildFilterJoinSelectCosmosQuery,
+  buildFindJoinCosmosQuery,
+  buildWhereCosmosQuery3,
+  logQuery
+} from "./Cosmos/query.js"
+import { StoreMaker } from "./service.js"
 import type {
-  Filter,
+  FilterArgs,
   FilterJoinSelect,
-  JoinFindFilter,
-  LegacyFilter,
   PersistenceModelType,
   StorageConfig,
   Store,
-  StoreConfig,
-  StoreWhereFilter,
-  SupportedValues
+  StoreConfig
 } from "./service.js"
-import { StoreMaker } from "./service.js"
 
-// TODO: Retry operation when running into RU limit.
+export function CosmosStoreLive(config: Config<StorageConfig>) {
+  return config.config.flatMap(makeCosmosStore).toLayer(StoreMaker)
+}
+
+class CosmosDbOperationError {
+  constructor(readonly message: string) {}
+} // TODO: Retry operation when running into RU limit.
 export function makeCosmosStore({ prefix }: StorageConfig) {
   return Effect.gen(function*($) {
     const { db } = yield* $(CosmosClient)
@@ -243,9 +252,13 @@ export function makeCosmosStore({ prefix }: StorageConfig) {
             /**
              * May return duplicate results for "join_find", when matching more than once.
              */
-            filter: (filter: Filter<PM>, cursor?: { skip?: number; limit?: number }) => {
-              const skip = cursor?.skip
-              const limit = cursor?.limit
+            filter: <U extends keyof PM = never>(
+              f: FilterArgs<PM, U>
+            ) => {
+              const skip = f?.skip
+              const limit = f?.limit
+              const filter = f.filter ?? { type: "new-kid", build: () => [] }
+              type M = U extends undefined ? PM : Pick<PM, U>
               return (filter.type === "join_find"
                 // This is a problem if one of the multiple joined arrays can be empty!
                 // https://stackoverflow.com/questions/60320780/azure-cosmosdb-sql-join-not-returning-results-when-the-child-contains-empty-arra
@@ -259,24 +272,33 @@ export function makeCosmosStore({ prefix }: StorageConfig) {
                         Effect.promise(() =>
                           container
                             .items
-                            .query<PM>(q)
+                            .query<M>(q)
                             .fetchAll()
                             .then(({ resources }) => resources)
                         )
                       )
                   )
                   .map((_) => _.flatMap((_) => _))
-                : Effect(buildCosmosQuery(filter, name, importedMarkerId, skip, limit))
+                : Effect(
+                  filter.type === "new-kid"
+                    ? buildWhereCosmosQuery3(
+                      filter.build(),
+                      name,
+                      importedMarkerId,
+                      f.select as string[] | undefined,
+                      skip,
+                      limit
+                    )
+                    : buildCosmosQuery(filter, name, importedMarkerId, skip, limit)
+                )
                   .tap((q) => logQuery(q))
                   .flatMap((q) =>
                     Effect.promise(() =>
                       container
                         .items
-                        .query<{ f: PM }>(
-                          q
-                        )
+                        .query<M>(q)
                         .fetchAll()
-                        .then(({ resources }) => resources.map((_) => _.f))
+                        .then(({ resources }) => resources)
                     )
                   ))
                 .withSpan("Cosmos.filter [effect-app/infra/Store]", {
@@ -389,231 +411,3 @@ export function makeCosmosStore({ prefix }: StorageConfig) {
     }
   })
 }
-
-function logQuery(q: {
-  query: string
-  parameters: {
-    name: string
-    value: SupportedValues | readonly SupportedValues[]
-  }[]
-}) {
-  return Effect
-    .logDebug("cosmos query")
-    .pipe(Effect.annotateLogs({
-      query: q.query,
-      parameters: JSON.stringify(
-        q.parameters.reduce((acc, v) => {
-          acc[v.name] = v.value
-          return acc
-        }, {} as Record<string, SupportedValues | readonly SupportedValues[]>),
-        undefined,
-        2
-      )
-    }))
-}
-
-/**
- * @deprecated: should build Select into Where query
- */
-export function buildFilterJoinSelectCosmosQuery(
-  filter: FilterJoinSelect,
-  k: string,
-  name: string,
-  skip?: number,
-  limit?: number
-) {
-  const lm = skip !== undefined || limit !== undefined ? `OFFSET ${skip ?? 0} LIMIT ${limit ?? 999999}` : ""
-  return {
-    query: `
-SELECT r, f.id as _rootId
-FROM ${name} f
-JOIN r IN f.${k}
-WHERE LOWER(r.${filter.valueKey}) = LOWER(@value)
-${lm}
-`,
-    parameters: [{ name: "@value", value: filter.value }]
-  }
-}
-
-/**
- * @deprecated: is now part of Where query as k.-1.valueKey
- */
-export function buildFindJoinCosmosQuery(
-  filter: JoinFindFilter,
-  k: string,
-  name: string,
-  skip?: number,
-  limit?: number
-) {
-  const lm = skip !== undefined || limit !== undefined ? `OFFSET ${skip ?? 0} LIMIT ${limit ?? 999999}` : ""
-  return {
-    query: `
-SELECT DISTINCT VALUE f
-FROM ${name} f
-JOIN r IN f.${k}
-WHERE LOWER(r.${filter.valueKey}) = LOWER(@value)
-${lm}`,
-    parameters: [{ name: "@value", value: filter.value }]
-  }
-}
-
-/**
- * @deprecated: should build into Where query
- */
-export function buildLegacyCosmosQuery<PM>(
-  filter: LegacyFilter<PM>,
-  name: string,
-  importedMarkerId: string,
-  skip?: number,
-  limit?: number
-) {
-  const lm = skip !== undefined || limit !== undefined ? `OFFSET ${skip ?? 0} LIMIT ${limit ?? 999999}` : ""
-  return {
-    query: `
-    SELECT f
-    FROM ${name} AS f
-    WHERE f.id != @id AND f.${
-      String(
-        filter.by
-      )
-    } LIKE @filter
-  ${lm}`,
-    parameters: [
-      { name: "@id", value: importedMarkerId },
-      {
-        name: "@filter",
-        value: filter.type === "endsWith"
-          ? `%${filter.value}`
-          : filter.type === "contains"
-          ? `%${filter.value}%`
-          : `${filter.value}%`
-      }
-    ]
-  }
-}
-
-export function buildWhereCosmosQuery(
-  filter: StoreWhereFilter,
-  name: string,
-  importedMarkerId: string,
-  skip?: number,
-  limit?: number
-) {
-  const lm = skip !== undefined || limit !== undefined ? `OFFSET ${skip ?? 0} LIMIT ${limit ?? 999999}` : ""
-  return {
-    query: `
-    SELECT f
-    FROM ${name} AS f
-    ${
-      filter
-        .where
-        .filter((_) => _.key.includes(".-1."))
-        .map((_) => _.key.split(".-1.")[0])
-        .map((_) => `JOIN ${_} IN f.${_}`)
-        .uniq(Equivalence.string)
-        .join("\n")
-    }
-    WHERE f.id != @id AND ${
-      filter
-        .where
-        .map((_) =>
-          _.key.includes(".-1.")
-            ? { ..._, f: _.key.split(".-1.")[0], key: _.key.split(".-1.")[1]! }
-            : { ..._, f: "f" }
-        )
-        .map(
-          (x, i) => {
-            const k = `${x.f}.${x.key}`
-            const v = `@v${i}`
-
-            switch (x.t) {
-              case "in":
-                return `ARRAY_CONTAINS(${v}, ${k})`
-              case "not-in":
-                return `(NOT ARRAY_CONTAINS(${v}, ${k}))`
-
-              case "includes":
-                return `ARRAY_CONTAINS(${k}, ${v})`
-              case "not-includes":
-                return `(NOT ARRAY_CONTAINS(${k}, ${v}))`
-              case "contains":
-                return `CONTAINS(${k}, ${v}, true)`
-              case "starts-with":
-                return `STARTSWITH(${k}, ${v}, true)`
-              case "ends-with":
-                return `ENDSWITH(${k}, ${v}, true)`
-              case "not-contains":
-                return `NOT(CONTAINS(${k}, ${v}, true))`
-              case "not-starts-with":
-                return `NOT(STARTSWITH(${k}, ${v}, true))`
-              case "not-ends-with":
-                return `NOT(ENDSWITH(${k}, ${v}, true))`
-            }
-
-            const lk = lowerIfNeeded(k, x.value)
-            const lv = lowerIfNeeded(v, x.value)
-
-            switch (x.t) {
-              case "lt":
-                return `${lk} < ${lv}`
-              case "lte":
-                return `${lk} <= ${lv}`
-              case "gt":
-                return `${lk} > ${lv}`
-              case "gte":
-                return `${lk} >= ${lv}`
-              case "not-eq":
-                return x.value === null
-                  ? `IS_NULL(${k}) = false`
-                  : `${lk} <> ${lv}`
-              case undefined:
-              case "eq":
-                return x.value === null
-                  ? `IS_NULL(${k}) = true`
-                  : `${lk} = ${lv}`
-            }
-          }
-        )
-        .join(filter.mode === "or" ? " OR " : " AND ")
-    }
-    ${lm}`,
-    parameters: [
-      { name: "@id", value: importedMarkerId },
-      ...filter
-        .where
-        .map((x, i) => ({
-          name: `@v${i}`,
-          value: x.value
-        }))
-    ]
-  }
-}
-
-// function isArray(t: SupportedValues | readonly SupportedValues[]): t is readonly SupportedValues[] {
-//   return Array.isArray(t)
-// }
-
-function lowerIfNeeded(key: unknown, value: unknown) {
-  return typeof value === "string" ? `LOWER(${key})` : `${key}`
-}
-
-export function buildCosmosQuery<PM>(
-  filter: LegacyFilter<PM> | StoreWhereFilter,
-  name: string,
-  importedMarkerId: string,
-  skip?: number,
-  limit?: number
-) {
-  return filter.type === "startsWith"
-      || filter.type === "endsWith"
-      || filter.type === "contains"
-    ? buildLegacyCosmosQuery(filter, name, importedMarkerId, skip, limit)
-    : buildWhereCosmosQuery(filter, name, importedMarkerId, skip, limit)
-}
-
-class CosmosDbOperationError {
-  constructor(readonly message: string) {}
-}
-export function CosmosStoreLive(config: Config<StorageConfig>) {
-  return config.config.flatMap(makeCosmosStore).toLayer(StoreMaker)
-}

package/_src/services/Store/Memory.ts

@@ -1,25 +1,33 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
+import { pick } from "@effect-app/prelude/utils"
 import type { RequestContext } from "../../RequestContext.js"
-import type { Filter, FilterJoinSelect, PersistenceModelType, Store, StoreConfig } from "./service.js"
+import type { FilterArgs, FilterJoinSelect, PersistenceModelType, Store, StoreConfig } from "./service.js"
 import { StoreMaker } from "./service.js"
 import { codeFilter, codeFilterJoinSelect, makeUpdateETag } from "./utils.js"
 
-export function memFilter<T extends { id: string }>(filter: Filter<T>, cursor?: { skip?: number; limit?: number }) {
-  return ((c: T[]): T[] => {
-    const skip = cursor?.skip
-    const limit = cursor?.limit
+export function memFilter<T extends PersistenceModelType<string>, U extends keyof T = never>(f: FilterArgs<T, U>) {
+  type M = U extends undefined ? T : Pick<T, U>
+  return ((c: T[]): M[] => {
+    const select = (r: T[]): M[] => (f.select ? r.map((_) => pick(_, f.select!)) : r) as any
+    const skip = f?.skip
+    const limit = f?.limit
     if (!skip && limit === 1) {
-      return c.findFirstMap(codeFilter(filter)).map(ReadonlyArray.make).getOrElse(() => [])
+      return select(
+        c.findFirstMap(f.filter ? codeFilter(f.filter) : (_) => Option.some(_)).map(ReadonlyArray.make).getOrElse(
+          () => []
+        )
+      )
     }
-    let r = c.filterMap(codeFilter(filter))
+    let r = f.filter ? c.filterMap(codeFilter(f.filter)) : c
     if (skip) {
       r = r.drop(skip)
     }
     if (limit !== undefined) {
       r = r.take(limit)
     }
-    return r
+
+    return select(r)
   })
 }
 
@@ -30,16 +38,11 @@ export const storeId = FiberRef.unsafeMake("primary")
  */
 export const restoreFromRequestContext = (ctx: RequestContext) => storeId.set(ctx.namespace ?? "primary")
 
-function logQuery(filter: any, cursor: any) {
+function logQuery(f: FilterArgs<any, any>) {
   return Effect
     .logDebug("mem query")
     .pipe(Effect.annotateLogs({
-      query: JSON.stringify(filter, undefined, 2),
-      cursor: JSON.stringify(
-        cursor,
-        undefined,
-        2
-      )
+      query: f
     }))
 }
 
@@ -91,10 +94,10 @@ export function makeMemoryStoreInt<Id extends string, PM extends PersistenceMode
               namespace
             }
           }),
-      filter: (filter: Filter<PM>, cursor?: { skip?: number; limit?: number }) =>
+      filter: (f) =>
         all
-          .tap(() => logQuery(filter, cursor))
-          .map(memFilter(filter, cursor))
+          .tap(() => logQuery(f))
+          .map(memFilter(f))
           .withSpan("Memory.filter [effect-app/infra/Store]", {
             attributes: { "repository.model_name": modelName, "repository.namespace": namespace }
           }),

package/_src/services/Store/Redis.ts

@@ -3,7 +3,7 @@ import { RedisClient } from "@effect-app/infra-adapters/redis-client"
 import { NotFoundError } from "../../errors.js"
 import { memFilter } from "./Memory.js"
 
-import type { Filter, FilterJoinSelect, PersistenceModelType, StorageConfig, Store, StoreConfig } from "./service.js"
+import type { FilterJoinSelect, PersistenceModelType, StorageConfig, Store, StoreConfig } from "./service.js"
 import { StoreMaker } from "./service.js"
 import { codeFilterJoinSelect, makeETag, makeUpdateETag } from "./utils.js"
 
@@ -62,8 +62,7 @@ export function makeRedisStore({ prefix }: StorageConfig) {
               .provideService(RedisClient, redis)
           const s: Store<PM, Id> = {
             all,
-            filter: (filter: Filter<PM>, cursor?: { skip?: number; limit?: number }) =>
-              all.map(memFilter(filter, cursor)),
+            filter: (f) => all.map(memFilter(f)),
             filterJoinSelect: <T extends object>(filter: FilterJoinSelect) =>
               all.map((c) => c.flatMap(codeFilterJoinSelect<PM, T>(filter))),
             find: (id) => asMap.map((_) => Option.fromNullable(_.get(id))),

package/_src/services/Store/codeFilter.test.ts

@@ -55,7 +55,7 @@ test("works", () => {
   expect(
     somethings
       .filterMap(codeFilter(
-        somethingsWhere((_) => _("b", Filters.$contains("b")))
+        somethingsWhere((_) => _("b", Filters.contains("b")))
       ))
       .toArray
   )
@@ -64,7 +64,7 @@ test("works", () => {
   expect(
     somethings
       .filterMap(codeFilter(
-        somethingsWhere((_) => _("b", Filters.$contains("2")))
+        somethingsWhere((_) => _("b", Filters.contains("2")))
       ))
       .toArray
   )
@@ -73,7 +73,7 @@ test("works", () => {
   expect(
     somethings
       .filterMap(codeFilter(
-        somethingsWhere((_) => _("b", Filters.$endsWith("b")))
+        somethingsWhere((_) => _("b", Filters.endsWith("b")))
       ))
       .toArray
   )
@@ -82,7 +82,7 @@ test("works", () => {
   expect(
     somethings
       .filterMap(codeFilter(
-        somethingsWhere((_) => _("b", Filters.$endsWith("2")))
+        somethingsWhere((_) => _("b", Filters.endsWith("2")))
       ))
       .toArray
   )
@@ -91,7 +91,7 @@ test("works", () => {
   expect(
     somethings
       .filterMap(codeFilter(
-        somethingsWhere((_) => _("b", Filters.$startsWith("b")))
+        somethingsWhere((_) => _("b", Filters.startsWith("b")))
       ))
       .toArray
   )
@@ -100,7 +100,7 @@ test("works", () => {
   expect(
     somethings
       .filterMap(codeFilter(
-        somethingsWhere((_) => _("b", Filters.$startsWith("2")))
+        somethingsWhere((_) => _("b", Filters.startsWith("2")))
       ))
       .toArray
   )
@@ -118,7 +118,7 @@ test("works", () => {
   expect(
     somethings
       .filterMap(codeFilter(
-        somethingsWhere((_) => _("c", Filters.$includes("c")))
+        somethingsWhere((_) => _("c", Filters.includes("c")))
       ))
       .toArray
   )
@@ -145,7 +145,7 @@ test("works", () => {
   expect(
     somethings
       .filterMap(codeFilter(
-        somethingsWhere((_) => _("d.-1.a", Filters.$isnt("a")))
+        somethingsWhere((_) => _("d.-1.a", Filters.isnt("a")))
       ))
       .toArray
   )

package/_src/services/Store/codeFilter.ts

@@ -0,0 +1,150 @@
+import { assertUnreachable, get } from "@effect-app/prelude/utils"
+import type { FilterR, FilterResult } from "./filterApi/query.js"
+import {
+  compareCaseInsensitive,
+  gtCaseInsensitive,
+  gteCaseInsensitive,
+  ltCaseInsensitive,
+  lteCaseInsensitive
+} from "./utils.js"
+
+export const codeFilterStatement = <E>(p: FilterR, x: E) => {
+  const k = get(x, p.path)
+  switch (p.op) {
+    case "in":
+      return p.value.includes(k)
+    case "notIn":
+      return !p.value.includes(k)
+    case "lt":
+      return ltCaseInsensitive(k, p.value)
+    case "lte":
+      return lteCaseInsensitive(k, p.value)
+    case "gt":
+      return gtCaseInsensitive(k, p.value)
+    case "gte":
+      return gteCaseInsensitive(k, p.value)
+    case "includes":
+      return (k as Array<string>).includes(p.value)
+    case "notIncludes":
+      return !(k as Array<string>).includes(p.value)
+    case "contains":
+      return (k as string).toLowerCase().includes(p.value.toLowerCase())
+    case "endsWith":
+      return (k as string).toLowerCase().endsWith(p.value.toLowerCase())
+    case "startsWith":
+      return (k as string).toLowerCase().startsWith(p.value.toLowerCase())
+    case "notContains":
+      return !(k as string).toLowerCase().includes(p.value.toLowerCase())
+    case "notEndsWith":
+      return !(k as string).toLowerCase().endsWith(p.value.toLowerCase())
+    case "notStartsWith":
+      return !(k as string).toLowerCase().startsWith(p.value.toLowerCase())
+    case "neq":
+      return p.path.includes(".-1.")
+        ? (get(x, p.path.split(".-1.")[0]) as any[])
+          // TODO: or vs and
+          .every((_) => !compareCaseInsensitive(get(_, p.path.split(".-1.")[1]!), p.value))
+        : !compareCaseInsensitive(k, p.value)
+    case "eq":
+    case undefined:
+      return p.path.includes(".-1.")
+        ? (get(x, p.path.split(".-1.")[0]) as any[])
+          // TODO: or vs and
+          .some((_) => compareCaseInsensitive(get(_, p.path.split(".-1.")[1]!), p.value))
+        : compareCaseInsensitive(k, p.value)
+    default: {
+      return assertUnreachable(p.op)
+    }
+  }
+}
+
+// const or = <E extends { id: string }, NE extends E>(filters: readonly FilterResult[], x: E): Option<NE> =>
+//   filters
+//       .some((p) => {
+//         if (p.t === "and-scope") {
+//           return and(p.result, x)
+//         }
+//         if (p.t === "or-scope") {
+//           return or(p.result, x)
+//         }
+//         if (p.t === "where-scope") {
+//           return codeFilter2(filters)(x)
+//         }
+//         return codeFilterStatement(p, x)
+//       })
+//     ? Option(x as unknown as NE)
+//     : Option.none
+
+// export const and = <E extends { id: string }, NE extends E>(filters: readonly FilterResult[], x: E): Option<NE> =>
+//   filters
+//       .every((p) => {
+//         if (p.t === "and-scope") {
+//           return and(p.result, x)
+//         }
+//         if (p.t === "or-scope") {
+//           return or(p.result, x)
+//         }
+//         if (p.t === "where-scope") {
+//           return codeFilter2(filters)(x)
+//         }
+//         return codeFilterStatement(p, x)
+//       })
+//     ? Option(x as unknown as NE)
+//     : Option.none
+
+// // TODO: how to handle and/or outside scopes.
+// // TODO: the scopes are not about and every, or some.. they're about logical grouping. It's a logical group + and/or etc.
+// // TODO: how to convert this to code? would we compile an actual function body instead?!
+// export function codeFilter2<E extends { id: string }, NE extends E>(filters: readonly FilterResult[]) {
+//   // TODO: handle or, and, or-scope, and-scope, where-scope
+//   return (x: E) =>
+//     // AND
+//     and<E, NE>(filters, x)
+//   // OR
+//   // or<E, NE>(filters, x)
+// }
+
+export const codeFilter3 = <E>(state: readonly FilterResult[]) => (sut: E) => codeFilter3_(state, sut)
+export const codeFilter3_ = <E>(state: readonly FilterResult[], sut: E): boolean => {
+  let s = ""
+  let l = 0
+  const printN = (n: number) => {
+    return n === 0 ? "" : ReadonlyArray.range(1, n).map(() => "  ").join("")
+  }
+  // TODO: path str updates
+
+  const process = (e: FilterR) => codeFilterStatement(e, sut)
+  for (const e of state) {
+    switch (e.t) {
+      case "where":
+        s += process(e)
+        break
+      case "or":
+        s += " || " + process(e)
+        break
+      case "and":
+        s += " && " + process(e)
+        break
+      case "or-scope": {
+        ;++l
+        s += ` || (\n${printN(l + 1)}${codeFilter3_(e.result, sut)}\n${printN(l)})`
+        ;--l
+        break
+      }
+      case "and-scope": {
+        ;++l
+        s += ` && (\n${printN(l + 1)}${codeFilter3_(e.result, sut)}\n${printN(l)})`
+        ;--l
+
+        break
+      }
+      case "where-scope": {
+        // ;++l
+        s += `(\n${printN(l + 1)}${codeFilter3_(e.result, sut)}\n)`
+        // ;--l
+        break
+      }
+    }
+  }
+  return eval(s)
+}