@adonis-agora/filter 0.7.0 → 0.8.0

package/skills/filter-querying/SKILL.md

@@ -0,0 +1,319 @@
+---
+name: filter-querying
+description: >
+  Advanced querying with @adonis-agora/filter beyond basic offset filtering —
+  keyset/cursor pagination (applyCursor, applyCursorFromRequest, buildCursorPage,
+  buildKeyset, opaque base64url encodeCursor/decodeCursor, after/before/first/last
+  params), the three search modes (searchable ILIKE scan vs fullText tsvector
+  websearch_to_tsquery/ts_rank vs vectorSimilarity pgvector distance metrics
+  cosine/l2/innerProduct with threshold/topK), DISTINCT projections and their
+  root-table-column restriction, and the richer request surface (parseSpatieRequest
+  with include/sparse fieldsets/cursor params, resolveInputFromRequest sources,
+  normalizeInput key casing). Use when serving stable deep pages, ranking by
+  relevance or embeddings, projecting distinct column values, or accepting
+  filters from a POST body.
+metadata:
+  type: core
+  library: "@adonis-agora/filter"
+  library_version: "0.7.0"
+  framework: adonisjs
+sources:
+  - "DavideCarvalho/adonis-filter:docs/pagination/index.mdx"
+  - "DavideCarvalho/adonis-filter:docs/search/index.mdx"
+  - "DavideCarvalho/adonis-filter:docs/search/full-text.mdx"
+  - "DavideCarvalho/adonis-filter:docs/search/vector-similarity.mdx"
+  - "DavideCarvalho/adonis-filter:docs/guides/lucid.mdx"
+  - "DavideCarvalho/adonis-filter:docs/definitions/request-input.mdx"
+---
+
+# Structured querying: cursors, search modes, distinct
+
+Offset pagination (`applyFilter` → `page`/`size`) drifts as rows are inserted
+and gets slow on deep pages. Keyset pagination seeks from a boundary row; text
+`search` routes through one of two engines you pick per policy; `distinct`
+projects the values that exist under the current filters.
+
+## Setup
+
+```typescript
+// app/controllers/users_controller.ts
+import type { HttpContext } from '@adonisjs/core/http'
+import { applyCursorFromRequest, buildCursorPage } from '@adonis-agora/filter'
+import { userFilter } from '#filters/user_filter'
+import User from '#models/user'
+
+export default class UsersController {
+  async index(ctx: HttpContext) {
+    const query = User.query()
+
+    // same spec as offset flow; cursor params instead of page/size
+    const resolved = applyCursorFromRequest(query, userFilter, ctx, { primaryKey: 'id' })
+
+    const rows = await query.exec()
+    const page = buildCursorPage(rows.map((r) => r.serialize()), resolved)
+
+    return {
+      data: page.items,
+      nextCursor: page.nextCursor,
+      prevCursor: page.prevCursor,
+      hasNext: page.hasNext,
+      hasPrev: page.hasPrev,
+    }
+  }
+}
+```
+
+Cursor params arrive as `after`/`before`/`first`/`last` (or JSON:API
+`page[after]`…). `after` wins if both bounds are sent. Filters, search, and sort
+pass through the same allow-list boundary as `applyFilter`.
+
+## Core patterns
+
+### 1. Keyset building blocks
+
+The effective sort plus a primary-key tiebreaker forms the keyset; the query is
+limited to `size + 1` so `buildCursorPage` can detect a next page:
+
+```typescript
+import { applyCursor, buildCursorPage } from '@adonis-agora/filter'
+
+const resolved = applyCursor(
+  User.query(),
+  { sort: [{ field: 'name', direction: 'asc' }], after: cursor, first: 20 },
+  { allowed: ['name'], primaryKey: 'id' },
+)
+
+const rows = await query.exec()
+const page = buildCursorPage(rows.map((r) => r.serialize()), resolved)
+```
+
+A malformed cursor is ignored, not fatal. Backward paging (`before`) reverses
+the walk internally and `buildCursorPage` restores your requested order;
+`prevCursor` from a forward page is a valid `before`.
+
+Source: `docs/pagination/index.mdx`
+
+### 2. Full-text search (keywords) via fullText
+
+When the policy declares `fullText`, the request `search` term routes through
+Postgres tsvector matching instead of ILIKE:
+
+```typescript
+import { defineFilter } from '@adonis-agora/filter'
+
+export const articleFilter = defineFilter({
+  filterable: ['status'],
+  fullText: {
+    column: 'search_vector', // stored, GIN-indexed tsvector column
+    columnKind: 'tsvector',
+    language: 'english',
+    rank: true, // ORDER BY ts_rank(...) DESC
+  },
+})
+```
+
+`?search=quick brown fox` becomes
+`WHERE "search_vector" @@ websearch_to_tsquery('english', ?)` — raw user input
+is safe because `websearch_to_tsquery` tolerates stray punctuation and the term
+always travels bound. Plain text columns work too (`columnKind: 'text'`),
+wrapped in `to_tsvector` at query time. `applyFullTextSearch` is exported for
+direct use outside the runner.
+
+Source: `docs/search/full-text.mdx`
+
+### 3. Vector similarity (embeddings) via vectorSimilarity
+
+Rank rows nearest-first by pgvector distance to a query embedding — supplied in
+code, never through the URL:
+
+```typescript
+export const docFilter = defineFilter({
+  filterable: ['status'],
+  vectorSimilarity: { column: 'embedding', metric: 'cosine', threshold: 0.25, topK: 20 },
+})
+
+// controller — compute the embedding, then pass it in options:
+const embedding = await embeddingService.embed(q)
+const { page, size } = applyFilterFromRequest(query, docFilter, ctx, {
+  vectorSimilarity: embedding,
+})
+```
+
+Similarity ordering lands before user sort (sort becomes the tiebreaker) and
+runs only when both policy and request carry an embedding. The embedding binds
+as `?::vector`; NaN/Infinity components throw rather than emitting bad SQL.
+`applyVectorSimilarity` is exported for direct use.
+
+Source: `docs/search/vector-similarity.mdx`
+
+### 4. DISTINCT projections come free on the wire format
+
+`?distinct=status` parses into `input.distinct`, is allow-listed like any
+field, and applies itself:
+
+```typescript
+const input = parseFilterRequest(request.qs())
+applyFilter(query, input, { allowed: ['status', 'baseId'] })
+const rows = await query.select('status') // DISTINCT already applied
+```
+
+### 5. Richer request shapes
+
+```typescript
+import {
+  parseSpatieRequest,
+  resolveInputFromRequest,
+  normalizeInput,
+} from '@adonis-agora/filter'
+
+// Spatie/JSON:API superset: includes, sparse fieldsets, cursor page params
+const input = parseSpatieRequest(request.qs())
+
+// source filters from body (POST /search), auto-merging query+body on writes
+const raw = resolveInputFromRequest(request, 'auto')
+
+// snake_case API → camelCase columns (top-level keys only)
+normalizeInput({ company_id: 5 }, { normalizer: 'camelCase' })
+```
+
+Source: `docs/definitions/request-input.mdx`
+
+## Common mistakes
+
+### [HIGH] Passing model instances to buildCursorPage
+
+Wrong:
+
+```typescript
+const rows = await query.exec()
+const page = buildCursorPage(rows, resolved) // Lucid model instances
+```
+
+Correct:
+
+```typescript
+const rows = await query.exec()
+const page = buildCursorPage(rows.map((r) => r.serialize()), resolved)
+```
+
+`buildCursorPage` reads keyset values off plain objects by field name; model
+instances do not expose those plain keys, so boundary cursors encode garbage
+and the next page silently starts from the wrong row.
+
+Source: docs/pagination/index.mdx ("Feed buildCursorPage plain rows")
+
+### [HIGH] Projecting distinct on a relation path
+
+Wrong:
+
+```typescript
+applyFilter(User.query(), input, { allowed: ['status', 'posts.title'] })
+// ?distinct=posts.title → dropped / InvalidColumnFilterError under throwOnInvalid
+```
+
+Correct:
+
+```typescript
+// distinct only names root-table columns: ?distinct=status
+applyFilter(User.query(), input, { allowed: ['status'] })
+```
+
+Relation filtering compiles to correlated EXISTS subqueries, so the relation is
+never joined and there is no alias to project from — the emitted SQL would be a
+missing-FROM error at the database. Adding the path to allow-lists cannot
+change this; it is structural, not a policy decision.
+
+Source: docs/guides/lucid.mdx ("distinct takes root-table columns only")
+
+### [HIGH] Expecting include/select to be applied
+
+Wrong:
+
+```typescript
+const input = parseSpatieRequest(request.qs())
+applyFilter(query, input, { allowed: ['name'] })
+return query.paginate(1, 25) // ?include=posts ignored — posts never loaded
+```
+
+Correct:
+
+```typescript
+const input = parseSpatieRequest(request.qs())
+applyFilter(query, input, { allowed: ['name'] })
+for (const relation of input.include ?? []) {
+  if (PRELOADABLE.includes(relation)) query.preload(relation as any)
+}
+return query.paginate(1, 25)
+```
+
+`include` and sparse fieldset `select` are parsed but deliberately applied by no
+runner — eager-loading which relations are safe stays your validated decision.
+The client builder's `.include(...)` is likewise only a request.
+
+Source: docs/guides/client.mdx ("include is a request, not an action"); docs/definitions/request-input.mdx
+
+### [HIGH] Confusing full-text search with vector similarity
+
+Wrong:
+
+```typescript
+defineFilter({ fullText: { column: 'embedding' } }) // tsvector @@ against a vector column
+```
+
+Correct:
+
+```typescript
+defineFilter({
+  vectorSimilarity: { column: 'embedding', metric: 'cosine' }, // embeddings rank nearest-first
+  // fullText: { column: 'search_vector' },                    // keywords match lexemes
+})
+```
+
+Full-text matches words against a tsvector document; vector similarity ranks by
+embedding distance. Configuring one while sending the other's input yields
+silently wrong search semantics. They are additive — filter by keywords, rank
+by embedding.
+
+Source: docs/search/index.mdx ("Full-text search ≠ vector similarity")
+
+### [MEDIUM] Trusting paginate() totals alongside distinct
+
+Wrong:
+
+```typescript
+const result = await query.distinct('status').paginate(page, size)
+result.totalCount // counts UN-deduped rows
+```
+
+Correct:
+
+```typescript
+// run your own countDistinct when the total must match the deduped page
+```
+
+Lucid's count leg rebuilds the SELECT, so the reported total counts un-deduped
+rows while the page itself is deduped.
+
+Source: docs/guides/lucid.mdx (distinct/paginate total callout)
+
+### [MEDIUM] Wrapping large text columns in query-time to_tsvector
+
+Wrong:
+
+```typescript
+fullText: { column: ['title', 'body'], columnKind: 'text' } // re-tokenizes every row
+```
+
+Correct:
+
+```typescript
+fullText: { column: 'search_vector', columnKind: 'tsvector' } // GIN-indexable
+```
+
+Query-time `to_tsvector` cannot use an index — fine for small tables, a
+sequential scan on anything big.
+
+Source: docs/search/full-text.mdx (text-columns warning)
+
+See also: `../filter-definitions/SKILL.md` (where fullText/vectorSimilarity are
+declared), `../filter-safety/SKILL.md` (the allow-list these paths respect).

package/skills/filter-safety/SKILL.md

@@ -0,0 +1,304 @@
+---
+name: filter-safety
+description: >
+  Lock down a public @adonis-agora/filter endpoint in AdonisJS. Covers the
+  allowed/sortable/searchable allow-lists as the security boundary, AllowList
+  forms ('*' | string[] | predicate), throwOnInvalid + InvalidColumnFilterError
+  mapped to HTTP 400, built-in structural validation (validateColumnFilters,
+  MAX_FILTER_DEPTH, field charset, operator aliases), query-string value
+  coercion with fieldTypes/coerceFilterValue, escapeLike for hand-built ILIKE
+  patterns, the trust boundary of low-level applyColumnFilters/applySort/
+  applySearch inside whereHas callbacks, layering VineJS for domain validation,
+  and unit-testing allow-listing with @adonis-agora/filter/testing
+  MockQueryBuilder. Use when exposing filters to untrusted clients, preventing
+  arbitrary-column probing, turning silent drops into 400s, or writing filter
+  unit tests.
+metadata:
+  type: core
+  library: "@adonis-agora/filter"
+  library_version: "0.7.0"
+  framework: adonisjs
+sources:
+  - "DavideCarvalho/adonis-filter:docs/guides/filter-classes.mdx"
+  - "DavideCarvalho/adonis-filter:docs/guides/validation.mdx"
+  - "DavideCarvalho/adonis-filter:docs/guides/operators.mdx"
+  - "DavideCarvalho/adonis-filter:docs/testing/index.mdx"
+  - "DavideCarvalho/adonis-filter:docs/guides/lucid.mdx"
+---
+
+# Filter safety: the allow-list is the security boundary
+
+`allowed`, `sortable`, and `searchable` are not conveniences — they are the only
+thing standing between a client query string and your columns. Any field a
+client references outside them is pruned before it reaches Lucid. This skill
+covers enforcing that boundary loudly, coercing hostile string values, and
+testing it without a database.
+
+## Setup
+
+```typescript
+// app/controllers/users_controller.ts
+import type { HttpContext } from '@adonisjs/core/http'
+import { parseFilterRequest, applyFilter } from '@adonis-agora/filter'
+import User from '#models/user'
+
+export default class UsersController {
+  async index({ request, response }: HttpContext) {
+    const input = parseFilterRequest(request.qs())
+    const query = User.query()
+
+    try {
+      const { page, size } = applyFilter(query, input, {
+        allowed: ['name', 'email', 'status'], // explicit array — never '*' on public endpoints
+        sortable: ['name', 'createdAt'],
+        searchable: ['name', 'email'],
+        maxSize: 100,
+        throwOnInvalid: true,
+      })
+      return query.paginate(page, size)
+    } catch (err) {
+      if (err instanceof InvalidColumnFilterError) {
+        return response.badRequest({ message: err.message })
+      }
+      throw err
+    }
+  }
+}
+```
+
+(Import `InvalidColumnFilterError` from `@adonis-agora/filter` alongside
+`parseFilterRequest` / `applyFilter`.)
+
+## Core patterns
+
+### 1. The AllowList has three forms
+
+`type AllowList = '*' | string[] | ((field) => boolean)`. Prefer an explicit
+array everywhere a client can reach; use the predicate form for rules a flat
+list cannot express — e.g. whitelisting one hop into `posts` under a cap:
+
+```typescript
+applyFilter(query, input, {
+  allowed: (field) => !field.includes('.') || /^posts\.[a-z]+$/i.test(field),
+})
+```
+
+The predicate receives the already alias-resolved target, never the client-facing alias key.
+
+Source: `docs/guides/filter-classes.mdx`
+
+### 2. throwOnInvalid — reject instead of silently dropping
+
+By default disallowed fields are dropped silently. With `throwOnInvalid: true`,
+both structural failures and allow-list rejections surface as
+`InvalidColumnFilterError`; handle it once globally instead of per controller:
+
+```typescript
+// app/exceptions/handler.ts
+import { InvalidColumnFilterError } from '@adonis-agora/filter'
+
+async handle(error: unknown, ctx: HttpContext) {
+  if (error instanceof InvalidColumnFilterError) {
+    return ctx.response.badRequest({ message: error.message })
+  }
+  return super.handle(error, ctx)
+}
+```
+
+Structural validation runs on every `applyFilter`: field charset
+(`^[a-zA-Z_][a-zA-Z0-9_.]*$`), known operators or SQL-symbol aliases
+(normalized in place), per-operator value shapes, and `AND`/`OR` nesting capped
+at `MAX_FILTER_DEPTH` (10). `validateColumnFilters` is exported to run the same
+checks manually.
+
+Source: `docs/guides/validation.mdx`
+
+### 3. Coerce query-string values with fieldTypes
+
+Query-string values are always strings. Declare kinds so bad values are
+rejected up front (dropped, or `InvalidColumnFilterError` → 400 under
+`throwOnInvalid`) instead of surfacing as Postgres cast errors:
+
+```typescript
+applyFilter(query, input, {
+  allowed: ['advisorId', 'dayOfWeek', 'isRecurring'],
+  fieldTypes: {
+    dayOfWeek: { kind: 'number' },
+    isRecurring: { kind: 'boolean' },
+  },
+})
+```
+
+Kinds: `'string' | 'number' | 'boolean' | 'date' | 'json' | 'unknown'`.
+Coercion rules worth knowing: an empty string is not a valid number;
+`null` passes every kind (legitimate `IS NULL`); a `date` string is validated
+but returned verbatim.
+
+Source: `docs/definitions/index.mdx` (Field types & value coercion)
+
+### 4. Escape LIKE patterns you build by hand
+
+All library string operators escape `%`, `_`, and `\` via `escapeLike()`
+before building patterns. If you build an ILIKE pattern yourself (outside the
+adapter), do the same:
+
+```typescript
+import { escapeLike } from '@adonis-agora/filter'
+
+qb.whereILike('code', `${escapeLike(prefix)}%`)
+```
+
+Source: `docs/guides/operators.mdx` (Security)
+
+### 5. Unit-test the boundary with the shipped mock
+
+`@adonis-agora/filter/testing` exports a recording `QueryBuilderLike` stand-in,
+so allow-listing is assertable with no database:
+
+```typescript
+import { test } from '@japa/runner'
+import { applyFilter } from '@adonis-agora/filter'
+import { makeMockQueryBuilder } from '@adonis-agora/filter/testing'
+
+test('drops disallowed fields', ({ assert }) => {
+  const qb = makeMockQueryBuilder()
+  applyFilter(
+    qb,
+    { filters: [
+      { field: 'name', operator: 'equals', value: 'Al' },
+      { field: 'secret', operator: 'equals', value: 'x' },
+    ] },
+    { allowed: ['name'] },
+  )
+  const flat = qb.flatten()
+  assert.deepInclude(flat, { method: 'where', args: ['name', 'Al'] })
+  assert.isUndefined(flat.find((c) => c.args.includes('secret')))
+})
+```
+
+Source: `docs/testing/index.mdx`, `docs/guides/testing.mdx`
+
+## Common mistakes
+
+### [CRITICAL] Allowing every column with `'*'`
+
+Wrong:
+
+```typescript
+applyFilter(query, input, { allowed: '*' })
+```
+
+Correct:
+
+```typescript
+applyFilter(query, input, { allowed: ['name', 'email', 'status'] })
+```
+
+A star allow-list disables column filtering entirely — any base column
+(`passwordHash`, internal flags) becomes probeable through filter values and
+result counts. Reserve it for trusted internal endpoints.
+
+Source: docs/guides/filter-classes.mdx ("The allow-list is the security boundary" warning)
+
+### [CRITICAL] Feeding client filters to low-level apply functions unpruned
+
+Wrong:
+
+```typescript
+const raw = parseFilterRequest(request.qs())
+query.whereHas('posts', (p) => applyColumnFilters(p, raw.filters ?? []))
+```
+
+Correct:
+
+```typescript
+const raw = parseFilterRequest(request.qs())
+// prune against a policy first (or validate fields yourself)…
+const scratch = User.query()
+applyFilter(scratch, raw, { allowed: ['title', 'status'] })
+query.whereHas('posts', (p) =>
+  applyColumnFilters(p, [{ field: 'title', operator: 'equals', value: 'Hi' }]),
+)
+```
+
+`applyColumnFilters` / `applySort` / `applySearch` trust their input — they
+enforce no allow-list — so client-derived filters handed straight to them can
+reference arbitrary columns inside the subquery.
+
+Source: docs/guides/lucid.mdx (low-level functions warning); docs/guides/relations.mdx
+
+### [HIGH] Trusting silent-drop while debugging
+
+Wrong:
+
+```typescript
+applyFilter(query, input, { allowed: ['name', 'createdAt'] })
+// ?sort=-craetedAt does nothing, no error anywhere
+```
+
+Correct:
+
+```typescript
+try {
+  const { page, size } = applyFilter(query, input, {
+    allowed: ['name', 'createdAt'],
+    throwOnInvalid: true,
+  })
+} catch (err) {
+  if (err instanceof InvalidColumnFilterError) {
+    return response.badRequest({ message: err.message })
+  }
+  throw err
+}
+```
+
+With defaults, a typo'd sort or filter just vanishes and the query "works" but
+ignores the client's intent — contract drift ships unnoticed.
+
+Source: docs/guides/validation.mdx (throwOnInvalid)
+
+### [HIGH] Assuming query-string numbers arrive as numbers
+
+Wrong:
+
+```typescript
+// ?filter[isRecurring]=xyz → Postgres: invalid input syntax for type boolean → 500
+applyFilter(query, input, { allowed: ['isRecurring'] })
+```
+
+Correct:
+
+```typescript
+applyFilter(query, input, {
+  allowed: ['isRecurring'],
+  fieldTypes: { isRecurring: { kind: 'boolean' } }, // rejected up front → drop or 400
+})
+```
+
+Every query-string value parses as a string; SQL implicit casts hide the gap
+until an uncastable value raises at the database as a user-input-driven 500.
+
+Source: docs/definitions/index.mdx (Field types & value coercion); docs/guides/operators.mdx
+
+### [MEDIUM] Building ILIKE patterns from unescaped user input
+
+Wrong:
+
+```typescript
+qb.whereILike('name', `%${userInput}%`) // userInput '%%' widens the match arbitrarily
+```
+
+Correct:
+
+```typescript
+import { escapeLike } from '@adonis-agora/filter'
+qb.whereILike('name', `%${escapeLike(userInput)}%`)
+```
+
+The adapter escapes `%`, `_` and `\` on every string operator it emits, but a
+pattern you interpolate yourself skips that protection entirely.
+
+Source: docs/guides/operators.mdx (Security — LIKE escaping)
+
+See also: `../filter-querying/SKILL.md` (cursor pagination and search route
+through this same boundary), `../filter-basics/SKILL.md` (endpoint wiring).

package/stubs/main.ts

@@ -0,0 +1,4 @@
+import { fileURLToPath } from 'node:url';
+
+/** Absolute path to this package's stubs, handed to `codemods.makeUsingStub`. */
+export const stubsRoot = fileURLToPath(new URL('./', import.meta.url));

package/stubs/make/filter/main.stub

@@ -0,0 +1,45 @@
+{{#var filterName = string(entity.name).pascalCase().removeSuffix('Filter').toString() + 'Filter'}}
+{{#var modelName = string(entity.name).pascalCase().removeSuffix('Filter').toString()}}
+{{#var filterFileName = string(entity.name).snakeCase().removeSuffix('filter').toString() + '_filter'}}
+{{#var modelFileName = string(entity.name).snakeCase().removeSuffix('filter').toString()}}
+{{{
+  exports({
+    to: app.makePath('app/filters', entity.path, filterFileName + '.ts')
+  })
+}}}
+import { BaseModelFilter } from '@adonis-agora/filter'
+import type { ModelQueryBuilderContract } from '@adonisjs/lucid/types/model'
+import {{ modelName }} from '#models/{{ modelFileName }}'
+
+export default class {{ filterName }} extends BaseModelFilter {
+  declare $query: ModelQueryBuilderContract<typeof {{ modelName }}>
+
+  static model = {{ modelName }}
+
+  /** Columns clients may filter on with the standard operators — these need no method. */
+  static filterable = ['name']
+  /** Columns clients may sort on. Defaults to `filterable`. */
+  static sortable = ['createdAt']
+  /** Columns the free-text `search` term scans. */
+  static searchable = ['name']
+
+  static defaultSort = [{ field: 'createdAt', direction: 'desc' as const }]
+  static defaultSize = 25
+  static maxSize = 100
+
+  /**
+   * Runs on every call, before anything the request asked for — the place for a scope the client
+   * cannot relax (a tenant, a soft-delete guard, an ownership check).
+   */
+  setup() {
+    // this.$query.whereNull('deletedAt')
+  }
+
+  /**
+   * One method per request key that needs SQL of its own. `?filter[fullName]=silva` lands here;
+   * writing the method IS what exposes the key, so it needs no allow-list entry.
+   */
+  // fullName(value: string) {
+  //   this.$query.whereILike('full_name', `%${value}%`)
+  // }
+}