@effect-app/infra 0.210.11 → 0.211.0

package/src/services/Store/Cosmos/query.ts

@@ -2,7 +2,7 @@ import { Effect, Equivalence, pipe, ReadonlyArray } from "effect-app"
 import type { NonEmptyReadonlyArray } from "effect-app"
 import { assertUnreachable } from "effect-app/utils"
 import type { FilterR, FilterResult } from "../filterApi/query.js"
-import type { FilterJoinSelect, JoinFindFilter, LegacyFilter, StoreWhereFilter, SupportedValues } from "../service.js"
+import type { SupportedValues } from "../service.js"
 
 export function logQuery(q: {
   query: string
@@ -26,208 +26,6 @@ export function logQuery(q: {
     }))
 }
 
-/**
- * @deprecated: should build Select into Where query
- */
-export function buildFilterJoinSelectCosmosQuery(
-  filter: FilterJoinSelect,
-  k: string,
-  name: string,
-  skip?: number,
-  limit?: number
-) {
-  const lm = skip !== undefined || limit !== undefined ? `OFFSET ${skip ?? 0} LIMIT ${limit ?? 999999}` : ""
-  return {
-    query: `
-SELECT r, f.id as _rootId
-FROM ${name} f
-JOIN r IN f.${k}
-WHERE r.${filter.valueKey} = @value
-${lm}
-`,
-    parameters: [{ name: "@value", value: filter.value }]
-  }
-}
-
-/**
- * @deprecated: is now part of Where query as k.-1.valueKey
- */
-export function buildFindJoinCosmosQuery(
-  filter: JoinFindFilter,
-  k: string,
-  name: string,
-  skip?: number,
-  limit?: number
-) {
-  const lm = skip !== undefined || limit !== undefined ? `OFFSET ${skip ?? 0} LIMIT ${limit ?? 999999}` : ""
-  return {
-    query: `
-SELECT DISTINCT VALUE f
-FROM ${name} f
-JOIN r IN f.${k}
-WHERE r.${filter.valueKey} = @value
-${lm}`,
-    parameters: [{ name: "@value", value: filter.value }]
-  }
-}
-
-/**
- * @deprecated: should build into Where query
- */
-export function buildLegacyCosmosQuery<PM>(
-  filter: LegacyFilter<PM>,
-  name: string,
-  importedMarkerId: string,
-  defaultValues: Record<string, unknown>,
-  skip?: number,
-  limit?: number
-) {
-  if (Object.keys(defaultValues).length) {
-    throw new Error("defaultValues not supported with legacy cosmos query: " + JSON.stringify(defaultValues))
-  }
-  const lm = skip !== undefined || limit !== undefined ? `OFFSET ${skip ?? 0} LIMIT ${limit ?? 999999}` : ""
-  return {
-    query: `
-    SELECT f
-    FROM ${name} f
-    WHERE f.id != @id AND f.${
-      String(
-        filter.by
-      )
-    } LIKE @filter
-  ${lm}`,
-    parameters: [
-      { name: "@id", value: importedMarkerId },
-      {
-        name: "@filter",
-        value: filter.type === "endsWith"
-          ? `%${filter.value}`
-          : filter.type === "contains"
-          ? `%${filter.value}%`
-          : `${filter.value}%`
-      }
-    ]
-  }
-}
-
-export function buildWhereCosmosQuery(
-  filter: StoreWhereFilter,
-  name: string,
-  importedMarkerId: string,
-  defaultValues: Record<string, unknown>,
-  skip?: number,
-  limit?: number
-) {
-  const lm = skip !== undefined || limit !== undefined ? `OFFSET ${skip ?? 0} LIMIT ${limit ?? 999999}` : ""
-
-  return {
-    query: `
-    SELECT f
-    FROM ${name} f
-    ${
-      pipe(
-        filter
-          .where
-          .filter((_) => _.key.includes(".-1."))
-          .map((_) => _.key.split(".-1.")[0])
-          .map((_) => `JOIN ${_} IN f.${_}`),
-        ReadonlyArray.dedupeWith(Equivalence.string)
-      )
-        .join("\n")
-    }
-    WHERE f.id != @id AND ${
-      filter
-        .where
-        .map((_) =>
-          _.key.includes(".-1.")
-            ? { ..._, f: _.key.split(".-1.")[0], key: _.key.split(".-1.")[1]! }
-            : { ..._, f: "f" }
-        )
-        .map(
-          (x, i) => {
-            let k = `${x.f}.${x.key}`
-            k = x.key in defaultValues ? `(${k} ?? ${JSON.stringify(defaultValues[x.key])})` : k
-            const v = `@v${i}`
-
-            switch (x.t) {
-              case "in":
-                return `ARRAY_CONTAINS(${v}, ${k})`
-              case "not-in":
-                return `(NOT ARRAY_CONTAINS(${v}, ${k}))`
-
-              case "includes":
-                return `ARRAY_CONTAINS(${k}, ${v})`
-              case "not-includes":
-                return `(NOT ARRAY_CONTAINS(${k}, ${v}))`
-              case "contains":
-                return `CONTAINS(${k}, ${v}, true)`
-              case "starts-with":
-                return `STARTSWITH(${k}, ${v}, true)`
-              case "ends-with":
-                return `ENDSWITH(${k}, ${v}, true)`
-              case "not-contains":
-                return `NOT(CONTAINS(${k}, ${v}, true))`
-              case "not-starts-with":
-                return `NOT(STARTSWITH(${k}, ${v}, true))`
-              case "not-ends-with":
-                return `NOT(ENDSWITH(${k}, ${v}, true))`
-            }
-
-            switch (x.t) {
-              case "lt":
-                return `${k} < ${v}`
-              case "lte":
-                return `${k} <= ${v}`
-              case "gt":
-                return `${k} > ${v}`
-              case "gte":
-                return `${k} >= ${v}`
-              case "not-eq":
-                return x.value === null
-                  ? `IS_NULL(${k}) = false`
-                  : `${k} <> ${v}`
-              case undefined:
-              case "eq":
-                return x.value === null
-                  ? `IS_NULL(${k}) = true`
-                  : `${k} = ${v}`
-            }
-          }
-        )
-        .join(filter.mode === "or" ? " OR " : " AND ")
-    }
-    ${lm}`,
-    parameters: [
-      { name: "@id", value: importedMarkerId },
-      ...filter
-        .where
-        .map((x, i) => ({
-          name: `@v${i}`,
-          value: x.value
-        }))
-    ]
-  }
-}
-
-// function isArray(t: SupportedValues | readonly SupportedValues[]): t is readonly SupportedValues[] {
-//   return Array.isArray(t)
-// }
-
-export function buildCosmosQuery<PM>(
-  filter: LegacyFilter<PM> | StoreWhereFilter,
-  name: string,
-  importedMarkerId: string,
-  defaultValues: Record<string, unknown>,
-  skip?: number,
-  limit?: number
-) {
-  return filter.type === "startsWith"
-      || filter.type === "endsWith"
-      || filter.type === "contains"
-    ? buildLegacyCosmosQuery(filter, name, importedMarkerId, defaultValues, skip, limit)
-    : buildWhereCosmosQuery(filter, name, importedMarkerId, defaultValues, skip, limit)
-}
-
 export function buildWhereCosmosQuery3(
   filter: readonly FilterResult[],
   name: string,

package/src/services/Store/Cosmos.ts

@@ -4,24 +4,11 @@ import { toNonEmptyArray } from "@effect-app/core/Array"
 import { CosmosClient, CosmosClientLayer } from "@effect-app/infra-adapters/cosmos-client"
 import { Chunk, Duration, Effect, Layer, Option, ReadonlyArray, Secret } from "effect-app"
 import type { NonEmptyReadonlyArray } from "effect-app"
-import { dropUndefinedT, omit, pick, spread } from "effect-app/utils"
+import { dropUndefinedT, omit, pick } from "effect-app/utils"
 import { OptimisticConcurrencyException } from "../../errors.js"
-import {
-  buildCosmosQuery,
-  buildFilterJoinSelectCosmosQuery,
-  buildFindJoinCosmosQuery,
-  buildWhereCosmosQuery3,
-  logQuery
-} from "./Cosmos/query.js"
+import { buildWhereCosmosQuery3, logQuery } from "./Cosmos/query.js"
 import { StoreMaker } from "./service.js"
-import type {
-  FilterArgs,
-  FilterJoinSelect,
-  PersistenceModelType,
-  StorageConfig,
-  Store,
-  StoreConfig
-} from "./service.js"
+import type { FilterArgs, PersistenceModelType, StorageConfig, Store, StoreConfig } from "./service.js"
 
 class CosmosDbOperationError {
   constructor(readonly message: string) {}
@@ -213,15 +200,14 @@ function makeCosmosStore({ prefix }: StorageConfig) {
                 parameters: [{ name: "@id", value: importedMarkerId }]
               }))
               .pipe(
-                Effect
-                  .tap((q) => logQuery(q)),
+                Effect.tap((q) => logQuery(q)),
                 Effect.flatMap((q) =>
                   Effect.promise(() =>
                     container
                       .items
                       .query<PM>(q)
                       .fetchAll()
-                      .then(({ resources }) => resources)
+                      .then(({ resources }) => resources.map((_) => ({ ...defaultValues, ..._ })))
                   )
                 ),
                 Effect
@@ -229,48 +215,6 @@ function makeCosmosStore({ prefix }: StorageConfig) {
                     attributes: { "repository.container_id": containerId, "repository.model_name": name }
                   })
               ),
-            filterJoinSelect: <T extends object>(
-              filter: FilterJoinSelect,
-              cursor?: { skip?: number; limit?: number }
-            ) =>
-              Effect
-                .forEach(
-                  filter
-                    .keys,
-                  (k) =>
-                    Effect
-                      .sync(() => buildFilterJoinSelectCosmosQuery(filter, k, name, cursor?.skip, cursor?.limit))
-                      .pipe(
-                        Effect
-                          .tap((q) => logQuery(q)),
-                        Effect.flatMap((q) =>
-                          Effect.promise(() =>
-                            container
-                              .items
-                              .query<T>(q)
-                              .fetchAll()
-                              .then(({ resources }) => resources.map((_) => ({ ...defaultValues, ..._ })))
-                          )
-                        )
-                      )
-                )
-                .pipe(
-                  Effect.map((a) => {
-                    const v = a
-                      .flatMap((_) => _)
-                      .map((x) =>
-                        spread(
-                          x,
-                          ({ r, ...rest }: any) => ({ ...rest, ...r } as T & { _rootId: string })
-                        )
-                      )
-                    return v
-                  }),
-                  Effect
-                    .withSpan("Cosmos.filterJoinSelect [effect-app/infra/Store]", {
-                      attributes: { "repository.container_id": containerId, "repository.model_name": name }
-                    })
-                ),
             /**
              * May return duplicate results for "join_find", when matching more than once.
              */
@@ -281,82 +225,40 @@ function makeCosmosStore({ prefix }: StorageConfig) {
               const limit = f?.limit
               const filter = f.filter ?? { type: "new-kid", build: () => [] }
               type M = U extends undefined ? PM : Pick<PM, U>
-              return (filter.type === "join_find"
-                // This is a problem if one of the multiple joined arrays can be empty!
-                // https://stackoverflow.com/questions/60320780/azure-cosmosdb-sql-join-not-returning-results-when-the-child-contains-empty-arra
-                // so we use multiple queries instead.
-                ? Effect
-                  .forEach(
-                    filter
-                      .keys,
-                    (k) =>
-                      Effect
-                        .sync(() => buildFindJoinCosmosQuery(filter, k, name, skip, limit))
-                        .pipe(
-                          Effect.tap((q) => logQuery(q)),
-                          Effect
-                            .flatMap((q) =>
-                              Effect.promise(() =>
-                                container
-                                  .items
-                                  .query<M>(q)
-                                  .fetchAll()
-                                  .then(({ resources }) => resources.map((_) => ({ ...defaultValues, ..._ })))
-                              )
-                            )
-                        )
+              return Effect
+                .sync(() =>
+                  buildWhereCosmosQuery3(
+                    filter.build(),
+                    name,
+                    importedMarkerId,
+                    defaultValues,
+                    f.select as NonEmptyReadonlyArray<string> | undefined,
+                    f.order as NonEmptyReadonlyArray<{ key: string; direction: "ASC" | "DESC" }> | undefined,
+                    skip,
+                    limit
                   )
-                  .pipe(Effect.map((_) => _.flatMap((_) => _)))
-                : (filter.type === "new-kid"
-                  ? Effect
-                    .sync(() =>
-                      buildWhereCosmosQuery3(
-                        filter.build(),
-                        name,
-                        importedMarkerId,
-                        defaultValues,
-                        f.select as NonEmptyReadonlyArray<string> | undefined,
-                        f.order as NonEmptyReadonlyArray<{ key: string; direction: "ASC" | "DESC" }> | undefined,
-                        skip,
-                        limit
+                )
+                .pipe(
+                  Effect.tap((q) => logQuery(q)),
+                  Effect
+                    .flatMap((q) =>
+                      Effect.promise(() =>
+                        f.select
+                          ? container
+                            .items
+                            .query<M>(q)
+                            .fetchAll()
+                            .then(({ resources }) =>
+                              resources.map((_) => ({ ...pick(defaultValues, f.select!), ..._ }))
+                            )
+                          : container
+                            .items
+                            .query<{ f: M }>(q)
+                            .fetchAll()
+                            .then(({ resources }) => resources.map((_) => ({ ...defaultValues, ..._.f })))
                       )
                     )
-                    .pipe(
-                      Effect.tap((q) => logQuery(q)),
-                      Effect
-                        .flatMap((q) =>
-                          Effect.promise(() =>
-                            f.select
-                              ? container
-                                .items
-                                .query<M>(q)
-                                .fetchAll()
-                                .then(({ resources }) =>
-                                  resources.map((_) => ({ ...pick(defaultValues, f.select!), ..._ }))
-                                )
-                              : container
-                                .items
-                                .query<{ f: M }>(q)
-                                .fetchAll()
-                                .then(({ resources }) => resources.map((_) => ({ ...defaultValues, ..._.f })))
-                          )
-                        )
-                    )
-                  : Effect
-                    .sync(() => buildCosmosQuery(filter, name, importedMarkerId, defaultValues, skip, limit))
-                    .pipe(
-                      Effect.tap((q) => logQuery(q)),
-                      Effect
-                        .flatMap((q) =>
-                          Effect.promise(() =>
-                            container
-                              .items
-                              .query<{ f: M }>(q)
-                              .fetchAll()
-                              .then(({ resources }) => resources.map((_) => ({ ...defaultValues, ..._.f })))
-                          )
-                        )
-                    )))
+                )
                 .pipe(Effect.withSpan("Cosmos.filter [effect-app/infra/Store]", {
                   attributes: { "repository.container_id": containerId, "repository.model_name": name }
                 }))
@@ -468,7 +370,8 @@ function makeCosmosStore({ prefix }: StorageConfig) {
               Effect.promise(() =>
                 container.items.create({
                   _partitionKey: importedMarkerId,
-                  id: importedMarkerId
+                  id: importedMarkerId,
+                  ttl: -1
                 })
               )
             )

package/src/services/Store/Disk.ts

@@ -150,7 +150,6 @@ export function makeDiskStore({ prefix }: StorageConfig, dir: string) {
             all: Effect.flatMap(getStore, (_) => _.all),
             find: (...args) => Effect.flatMap(getStore, (_) => _.find(...args)),
             filter: (...args) => Effect.flatMap(getStore, (_) => _.filter(...args)),
-            filterJoinSelect: (...args) => Effect.flatMap(getStore, (_) => _.filterJoinSelect(...args)),
             set: (...args) => Effect.flatMap(getStore, (_) => _.set(...args)),
             batchSet: (...args) => Effect.flatMap(getStore, (_) => _.batchSet(...args)),
             bulkSet: (...args) => Effect.flatMap(getStore, (_) => _.bulkSet(...args)),

package/src/services/Store/Memory.ts

@@ -4,9 +4,9 @@ import { Effect, FiberRef, flow, Option, Order, pipe, ReadonlyArray, Ref } from
 import type { NonEmptyArray, NonEmptyReadonlyArray } from "effect-app"
 import { get, pick } from "effect-app/utils"
 import type { RequestContext } from "../../RequestContext.js"
-import type { FilterArgs, FilterJoinSelect, PersistenceModelType, Store, StoreConfig } from "./service.js"
+import type { FilterArgs, PersistenceModelType, Store, StoreConfig } from "./service.js"
 import { StoreMaker } from "./service.js"
-import { codeFilter, codeFilterJoinSelect, makeUpdateETag } from "./utils.js"
+import { codeFilter, makeUpdateETag } from "./utils.js"
 
 export function memFilter<T extends PersistenceModelType<string>, U extends keyof T = never>(f: FilterArgs<T, U>) {
   type M = U extends undefined ? T : Pick<T, U>
@@ -149,20 +149,6 @@ export function makeMemoryStoreInt<Id extends string, PM extends PersistenceMode
               attributes: { "repository.model_name": modelName, "repository.namespace": namespace }
             })
           ),
-      filterJoinSelect: <T extends object>(filter: FilterJoinSelect) =>
-        all
-          .pipe(
-            Effect.map((c) => c.flatMap(codeFilterJoinSelect<PM, T>(filter))),
-            Effect
-              .withSpan(
-                "Memory.filterJoinSelect [effect-app/infra/Store]",
-                {
-                  attributes: {
-                    modelName
-                  }
-                }
-              )
-          ),
       set: (e) =>
         s
           .find(e.id)
@@ -257,7 +243,6 @@ export const makeMemoryStore = () => ({
         all: Effect.flatMap(getStore, (_) => _.all),
         find: (...args) => Effect.flatMap(getStore, (_) => _.find(...args)),
         filter: (...args) => Effect.flatMap(getStore, (_) => _.filter(...args)),
-        filterJoinSelect: (...args) => Effect.flatMap(getStore, (_) => _.filterJoinSelect(...args)),
         set: (...args) => Effect.flatMap(getStore, (_) => _.set(...args)),
         batchSet: (...args) => Effect.flatMap(getStore, (_) => _.batchSet(...args)),
         bulkSet: (...args) => Effect.flatMap(getStore, (_) => _.bulkSet(...args)),

package/src/services/Store/Redis.ts.bak

@@ -4,9 +4,9 @@ import { Effect, Layer, Option, ReadonlyArray, Secret } from "effect-app"
 import type { NonEmptyArray, NonEmptyReadonlyArray } from "effect-app"
 import { NotFoundError } from "../../errors.js"
 import { memFilter } from "./Memory.js"
-import type { FilterJoinSelect, PersistenceModelType, StorageConfig, Store, StoreConfig } from "./service.js"
+import type { PersistenceModelType, StorageConfig, Store, StoreConfig } from "./service.js"
 import { StoreMaker } from "./service.js"
-import { codeFilterJoinSelect, makeETag, makeUpdateETag } from "./utils.js"
+import { makeETag, makeUpdateETag } from "./utils.js"
 
 function makeRedisStore({ prefix }: StorageConfig) {
   return Effect.gen(function*($) {
@@ -61,8 +61,6 @@ function makeRedisStore({ prefix }: StorageConfig) {
           const s: Store<PM, Id> = {
             all,
             filter: (f) => all.map(memFilter(f)),
-            filterJoinSelect: <T extends object>(filter: FilterJoinSelect) =>
-              all.map((c) => c.flatMap(codeFilterJoinSelect<PM, T>(filter))),
             find: (id) => asMap.map((_) => Option.fromNullable(_.get(id))),
             set: (e) =>
               s

package/src/services/Store/filterApi/query.ts

@@ -210,11 +210,11 @@ export type Filts<TFieldValues extends FieldValues> = {
   ): QueryBuilder<TFieldValues>
   <
     TFieldName extends FieldPath<TFieldValues>,
-    V extends FieldPathValue<TFieldValues, TFieldName>
+    V extends string
   >(
     path: TFieldName,
     op: "startsWith" | "endsWith" | "contains" | "notContains" | "notStartsWith" | "notEndsWith",
-    value: V // only strings?
+    value: V
   ): QueryBuilder<TFieldValues>
   <
     TFieldName extends FieldPath<TFieldValues>,

package/src/services/Store/service.ts

@@ -34,37 +34,7 @@ export type Where =
     value: readonly (SupportedValues)[]
   }
 
-// default is where
-export type StoreWhereFilter = {
-  type?: "where" | undefined
-  mode?: "and" | "or" | undefined // default is and
-  where: readonly [
-    Where,
-    ...(Where[])
-  ]
-}
-export type LegacyFilter<E> =
-  | { by: keyof E; type: "startsWith"; value: any }
-  | { by: keyof E; type: "endsWith"; value: any }
-  | { by: keyof E; type: "contains"; value: any }
-export type JoinFindFilter = {
-  type: "join_find"
-  keys: readonly string[] /* value paths of E */
-  valueKey: string /* value paths of E[keys][valueKey] */
-  value: any /* value path[valueKey] of E */
-}
-export type Filter<E extends FieldValues> =
-  | JoinFindFilter
-  | StoreWhereFilter
-  | LegacyFilter<E>
-  | QueryBuilder<E>
-
-export type FilterJoinSelect = {
-  type: "filter_join_select"
-  keys: readonly string[] /* value paths of E */
-  valueKey: string /* value paths of E[keys][valueKey] */
-  value: any /* value path[valueKey] of E */
-}
+export type Filter<E extends FieldValues> = QueryBuilder<E>
 
 export interface O<PM extends PersistenceModelType<unknown>> {
   key: keyof PM
@@ -86,10 +56,6 @@ export type FilterFunc<PM extends PersistenceModelType<unknown>> = <U extends ke
 export interface Store<PM extends PersistenceModelType<Id>, Id> {
   all: Effect<PM[]>
   filter: FilterFunc<PM>
-  /** @deprecated */
-  filterJoinSelect: <T extends object>(
-    filter: FilterJoinSelect
-  ) => Effect<(T & { _rootId: string })[]>
   find: (id: Id) => Effect<Option<PM>>
   set: (e: PM) => Effect<PM, OptimisticConcurrencyException>
   batchSet: (