@tinybirdco/sdk 0.0.44 → 0.0.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -8
- package/dist/api/api.d.ts.map +1 -1
- package/dist/api/api.js +8 -3
- package/dist/api/api.js.map +1 -1
- package/dist/api/api.test.js +24 -6
- package/dist/api/api.test.js.map +1 -1
- package/dist/cli/commands/migrate.d.ts.map +1 -1
- package/dist/cli/commands/migrate.js +4 -3
- package/dist/cli/commands/migrate.js.map +1 -1
- package/dist/cli/commands/migrate.test.js +42 -4
- package/dist/cli/commands/migrate.test.js.map +1 -1
- package/dist/client/base.d.ts +1 -1
- package/dist/client/base.js +1 -1
- package/dist/generator/connection.d.ts +1 -1
- package/dist/generator/connection.d.ts.map +1 -1
- package/dist/generator/connection.js +25 -2
- package/dist/generator/connection.js.map +1 -1
- package/dist/generator/connection.test.js +37 -14
- package/dist/generator/connection.test.js.map +1 -1
- package/dist/generator/datasource.d.ts.map +1 -1
- package/dist/generator/datasource.js +23 -0
- package/dist/generator/datasource.js.map +1 -1
- package/dist/generator/datasource.test.js +53 -9
- package/dist/generator/datasource.test.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/infer/index.d.ts +3 -3
- package/dist/migrate/emit-ts.d.ts.map +1 -1
- package/dist/migrate/emit-ts.js +69 -13
- package/dist/migrate/emit-ts.js.map +1 -1
- package/dist/migrate/parse-connection.d.ts +2 -2
- package/dist/migrate/parse-connection.d.ts.map +1 -1
- package/dist/migrate/parse-connection.js +61 -18
- package/dist/migrate/parse-connection.js.map +1 -1
- package/dist/migrate/parse-datasource.d.ts.map +1 -1
- package/dist/migrate/parse-datasource.js +31 -0
- package/dist/migrate/parse-datasource.js.map +1 -1
- package/dist/migrate/types.d.ts +18 -1
- package/dist/migrate/types.d.ts.map +1 -1
- package/dist/schema/connection.d.ts +49 -6
- package/dist/schema/connection.d.ts.map +1 -1
- package/dist/schema/connection.js +44 -9
- package/dist/schema/connection.js.map +1 -1
- package/dist/schema/connection.test.js +72 -17
- package/dist/schema/connection.test.js.map +1 -1
- package/dist/schema/datasource.d.ts +16 -1
- package/dist/schema/datasource.d.ts.map +1 -1
- package/dist/schema/datasource.js +3 -0
- package/dist/schema/datasource.js.map +1 -1
- package/dist/schema/datasource.test.js +21 -0
- package/dist/schema/datasource.test.js.map +1 -1
- package/dist/schema/params.d.ts +3 -3
- package/dist/schema/params.d.ts.map +1 -1
- package/dist/schema/params.js +3 -3
- package/dist/schema/params.js.map +1 -1
- package/dist/schema/project.d.ts +3 -3
- package/dist/schema/project.js +3 -3
- package/dist/schema/types.d.ts +8 -8
- package/dist/schema/types.d.ts.map +1 -1
- package/dist/schema/types.js +4 -4
- package/dist/schema/types.js.map +1 -1
- package/package.json +1 -1
- package/src/api/api.test.ts +32 -6
- package/src/api/api.ts +14 -3
- package/src/cli/commands/migrate.test.ts +58 -4
- package/src/cli/commands/migrate.ts +6 -4
- package/src/client/base.ts +1 -1
- package/src/generator/connection.test.ts +45 -14
- package/src/generator/connection.ts +30 -2
- package/src/generator/datasource.test.ts +61 -9
- package/src/generator/datasource.ts +38 -1
- package/src/index.ts +12 -1
- package/src/infer/index.ts +3 -3
- package/src/migrate/emit-ts.ts +80 -16
- package/src/migrate/parse-connection.ts +108 -30
- package/src/migrate/parse-datasource.ts +46 -1
- package/src/migrate/types.ts +24 -2
- package/src/schema/connection.test.ts +92 -17
- package/src/schema/connection.ts +86 -10
- package/src/schema/datasource.test.ts +25 -0
- package/src/schema/datasource.ts +21 -1
- package/src/schema/params.ts +3 -3
- package/src/schema/project.ts +3 -3
- package/src/schema/types.ts +10 -10
package/dist/schema/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/schema/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,iEAAiE;AACjE,+DAA+D;AAC/D,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;AA4CzD,SAAS,eAAe,CACtB,YAA2B,EAC3B,YAA2B,EAAE;IAE7B,MAAM,SAAS,GAAuD;QACpE,CAAC,eAAe,CAAC,EAAE,IAAI;QACvB,KAAK,EAAE,SAA6B;QACpC,aAAa,EAAE,YAAY;QAC3B,UAAU,EAAE,SAAS;QACrB,YAAY;QACZ,SAAS;QAET,QAAQ;YACN,iEAAiE;YACjE,oFAAoF;YACpF,IAAI,SAAS,CAAC,cAAc,EAAE,CAAC;gBAC7B,mFAAmF;gBACnF,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,0BAA0B,EAAE,IAAI,CAAC,CAAC;gBACxE,MAAM,OAAO,GAAG,2BAA2B,QAAQ,IAAI,CAAC;gBACxD,OAAO,eAAe,CACpB,OAAgD,EAChD,EAAE,GAAG,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE,CAC2E,CAAC;YAChH,CAAC;YACD,OAAO,eAAe,CACpB,YAAY,YAAY,GAAmC,EAC3D,EAAE,GAAG,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE,CACgE,CAAC;QACrG,CAAC;QAED,cAAc;YACZ,2DAA2D;YAC3D,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;gBACvB,6EAA6E;gBAC7E,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC;gBAClE,MAAM,OAAO,GAAG,2BAA2B,QAAQ,IAAI,CAAC;gBACxD,OAAO,eAAe,CACpB,OAAgD,EAChD,EAAE,GAAG,SAAS,EAAE,cAAc,EAAE,IAAI,EAAE,CAC0E,CAAC;YACrH,CAAC;YACD,OAAO,eAAe,CACpB,kBAAkB,YAAY,GAAyC,EACvE,EAAE,GAAG,SAAS,EAAE,cAAc,EAAE,IAAI,EAAE,CAC+D,CAAC;QAC1G,CAAC;QAED,OAAO,CAAC,KAAY;YAClB,OAAO,eAAe,CAAuB,YAAY,EAAE;gBACzD,GAAG,SAAS;gBACZ,UAAU,EAAE,IAAI;gBAChB,YAAY,EAAE,KAAK;aACpB,CAAmG,CAAC;QACvG,CAAC;QAED,KAAK,CAAC,KAAa;YACjB,OAAO,eAAe,CAAuB,YAAY,EAAE;gBACzD,GAAG,SAAS;gBACZ,KAAK;aACN,CAA2E,CAAC;QAC/E,CAAC;KACF,CAAC;IAEF,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,CAAC,GAAG;IACf,yCAAyC;IAEzC,iDAAiD;IACjD,MAAM,EAAE,GAAG,EAAE,CAAC,eAAe,CAAmB,QAAQ,CAAC;IAEzD,mEAAmE;IACnE,WAAW,EAAE,CAAC,MAAc,EAAE,EAAE,CAC9B,eAAe,CAAmC,eAAe,MAAM,GAAG,CAAC;IAE7E,mDAAmD;IACnD,IAAI,EAAE,GAAG,EAAE,CAAC,eAAe,CAAiB,MAAM,CAAC;IAEnD,0CAA0C;IAE1C,gDAAgD;IAChD,IAAI,EAAE,GAAG,EAAE,CAAC,eAAe,CAAiB,MAAM,CAAC;IAEnD,oCAAoC;IACpC,KAAK,EAAE,GAAG,EAAE,CAAC,eAAe,CAAkB,OAAO,CAAC;IAEtD,oCAAoC;IACpC,KAAK,EAAE,GAAG,EAAE,CAAC,eAAe,CAAkB,OAAO,CAAC;IAEtD,gFAAgF;IAChF,KAAK,EAAE,GAAG,EAAE,CAAC,eAAe,CAAkB,OAAO,CAAC;IAEtD,8DAA8D;IAC9D,MAAM,EAAE,GAAG,EAAE,CAAC,eAAe,CAAmB,QAAQ,CAAC;IAEzD,8DAA8D;IAC9D,MAAM,EAAE,GAAG,EAAE,CAAC,eAAe,CAAmB,QAAQ,CAAC;IAEzD,gDAAgD;IAChD,KAAK,EAAE,GAAG,EAAE,CAAC,eAAe,CAAkB,OAAO,CAAC;IAEtD,uCAAuC;IACvC,MAAM,EAAE,GAAG,EAAE,CAAC,eAAe,CAAmB,QAAQ,CAAC;IAEzD,uCAAuC;IACvC,MAAM,EAAE,GAAG,EAAE,CAAC,eAAe,CAAmB,QAAQ,CAAC;IAEzD,mFAAmF;IACnF,MAAM,EAAE,GAAG,EAAE,CAAC,eAAe,CAAmB,QAAQ,CAAC;IAEzD,iEAAiE;IACjE,OAAO,EAAE,GAAG,EAAE,CAAC,eAAe,CAAoB,SAAS,CAAC;IAE5D,iEAAiE;IACjE,OAAO,EAAE,GAAG,EAAE,CAAC,eAAe,CAAoB,SAAS,CAAC;IAE5D,wCAAwC;IAExC,sCAAsC;IACtC,OAAO,EAAE,GAAG,EAAE,CAAC,eAAe,CAAoB,SAAS,CAAC;IAE5D,yDAAyD;IACzD,OAAO,EAAE,GAAG,EAAE,CAAC,eAAe,CAAoB,SAAS,CAAC;IAE5D,6DAA6D;IAC7D,OAAO,EAAE,CAAC,SAAiB,EAAE,KAAa,EAAE,EAAE,CAC5C,eAAe,CACb,WAAW,SAAS,KAAK,KAAK,GAAG,CAClC;IAEH,oCAAoC;IAEpC,wCAAwC;IACxC,IAAI,EAAE,GAAG,EAAE,CAAC,eAAe,CAAkB,MAAM,CAAC;IAEpD,4CAA4C;IAE5C,
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/schema/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,iEAAiE;AACjE,+DAA+D;AAC/D,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;AA4CzD,SAAS,eAAe,CACtB,YAA2B,EAC3B,YAA2B,EAAE;IAE7B,MAAM,SAAS,GAAuD;QACpE,CAAC,eAAe,CAAC,EAAE,IAAI;QACvB,KAAK,EAAE,SAA6B;QACpC,aAAa,EAAE,YAAY;QAC3B,UAAU,EAAE,SAAS;QACrB,YAAY;QACZ,SAAS;QAET,QAAQ;YACN,iEAAiE;YACjE,oFAAoF;YACpF,IAAI,SAAS,CAAC,cAAc,EAAE,CAAC;gBAC7B,mFAAmF;gBACnF,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,0BAA0B,EAAE,IAAI,CAAC,CAAC;gBACxE,MAAM,OAAO,GAAG,2BAA2B,QAAQ,IAAI,CAAC;gBACxD,OAAO,eAAe,CACpB,OAAgD,EAChD,EAAE,GAAG,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE,CAC2E,CAAC;YAChH,CAAC;YACD,OAAO,eAAe,CACpB,YAAY,YAAY,GAAmC,EAC3D,EAAE,GAAG,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE,CACgE,CAAC;QACrG,CAAC;QAED,cAAc;YACZ,2DAA2D;YAC3D,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;gBACvB,6EAA6E;gBAC7E,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC;gBAClE,MAAM,OAAO,GAAG,2BAA2B,QAAQ,IAAI,CAAC;gBACxD,OAAO,eAAe,CACpB,OAAgD,EAChD,EAAE,GAAG,SAAS,EAAE,cAAc,EAAE,IAAI,EAAE,CAC0E,CAAC;YACrH,CAAC;YACD,OAAO,eAAe,CACpB,kBAAkB,YAAY,GAAyC,EACvE,EAAE,GAAG,SAAS,EAAE,cAAc,EAAE,IAAI,EAAE,CAC+D,CAAC;QAC1G,CAAC;QAED,OAAO,CAAC,KAAY;YAClB,OAAO,eAAe,CAAuB,YAAY,EAAE;gBACzD,GAAG,SAAS;gBACZ,UAAU,EAAE,IAAI;gBAChB,YAAY,EAAE,KAAK;aACpB,CAAmG,CAAC;QACvG,CAAC;QAED,KAAK,CAAC,KAAa;YACjB,OAAO,eAAe,CAAuB,YAAY,EAAE;gBACzD,GAAG,SAAS;gBACZ,KAAK;aACN,CAA2E,CAAC;QAC/E,CAAC;KACF,CAAC;IAEF,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,CAAC,GAAG;IACf,yCAAyC;IAEzC,iDAAiD;IACjD,MAAM,EAAE,GAAG,EAAE,CAAC,eAAe,CAAmB,QAAQ,CAAC;IAEzD,mEAAmE;IACnE,WAAW,EAAE,CAAC,MAAc,EAAE,EAAE,CAC9B,eAAe,CAAmC,eAAe,MAAM,GAAG,CAAC;IAE7E,mDAAmD;IACnD,IAAI,EAAE,GAAG,EAAE,CAAC,eAAe,CAAiB,MAAM,CAAC;IAEnD,0CAA0C;IAE1C,gDAAgD;IAChD,IAAI,EAAE,GAAG,EAAE,CAAC,eAAe,CAAiB,MAAM,CAAC;IAEnD,oCAAoC;IACpC,KAAK,EAAE,GAAG,EAAE,CAAC,eAAe,CAAkB,OAAO,CAAC;IAEtD,oCAAoC;IACpC,KAAK,EAAE,GAAG,EAAE,CAAC,eAAe,CAAkB,OAAO,CAAC;IAEtD,gFAAgF;IAChF,KAAK,EAAE,GAAG,EAAE,CAAC,eAAe,CAAkB,OAAO,CAAC;IAEtD,8DAA8D;IAC9D,MAAM,EAAE,GAAG,EAAE,CAAC,eAAe,CAAmB,QAAQ,CAAC;IAEzD,8DAA8D;IAC9D,MAAM,EAAE,GAAG,EAAE,CAAC,eAAe,CAAmB,QAAQ,CAAC;IAEzD,gDAAgD;IAChD,KAAK,EAAE,GAAG,EAAE,CAAC,eAAe,CAAkB,OAAO,CAAC;IAEtD,uCAAuC;IACvC,MAAM,EAAE,GAAG,EAAE,CAAC,eAAe,CAAmB,QAAQ,CAAC;IAEzD,uCAAuC;IACvC,MAAM,EAAE,GAAG,EAAE,CAAC,eAAe,CAAmB,QAAQ,CAAC;IAEzD,mFAAmF;IACnF,MAAM,EAAE,GAAG,EAAE,CAAC,eAAe,CAAmB,QAAQ,CAAC;IAEzD,iEAAiE;IACjE,OAAO,EAAE,GAAG,EAAE,CAAC,eAAe,CAAoB,SAAS,CAAC;IAE5D,iEAAiE;IACjE,OAAO,EAAE,GAAG,EAAE,CAAC,eAAe,CAAoB,SAAS,CAAC;IAE5D,wCAAwC;IAExC,sCAAsC;IACtC,OAAO,EAAE,GAAG,EAAE,CAAC,eAAe,CAAoB,SAAS,CAAC;IAE5D,yDAAyD;IACzD,OAAO,EAAE,GAAG,EAAE,CAAC,eAAe,CAAoB,SAAS,CAAC;IAE5D,6DAA6D;IAC7D,OAAO,EAAE,CAAC,SAAiB,EAAE,KAAa,EAAE,EAAE,CAC5C,eAAe,CACb,WAAW,SAAS,KAAK,KAAK,GAAG,CAClC;IAEH,oCAAoC;IAEpC,wCAAwC;IACxC,IAAI,EAAE,GAAG,EAAE,CAAC,eAAe,CAAkB,MAAM,CAAC;IAEpD,4CAA4C;IAE5C,2DAA2D;IAC3D,IAAI,EAAE,GAAG,EAAE,CAAC,eAAe,CAAiB,MAAM,CAAC;IAEnD,kFAAkF;IAClF,MAAM,EAAE,GAAG,EAAE,CAAC,eAAe,CAAmB,QAAQ,CAAC;IAEzD,iFAAiF;IACjF,QAAQ,EAAE,CAAC,QAAiB,EAAE,EAAE,CAC9B,QAAQ;QACN,CAAC,CAAC,eAAe,CAAkC,aAAa,QAAQ,IAAI,CAAC;QAC7E,CAAC,CAAC,eAAe,CAAqB,UAAU,CAAC;IAErD,kGAAkG;IAClG,UAAU,EAAE,CAAC,YAAmD,CAAC,EAAE,QAAiB,EAAE,EAAE,CACtF,QAAQ;QACN,CAAC,CAAC,eAAe,CACb,cAAc,SAAS,MAAM,QAAQ,IAAI,CAC1C;QACH,CAAC,CAAC,eAAe,CAAkC,cAAc,SAAS,GAAG,CAAC;IAElF,0CAA0C;IAE1C,6CAA6C;IAC7C,KAAK,EAAE,CACL,OAAiB,EAKjB,EAAE,CACF,eAAe,CACb,SAAS,OAAO,CAAC,aAAa,GAA4C,CAC3E;IAEH,wDAAwD;IACxD,KAAK,EAAE,CACL,GAAG,QAAmB,EAKtB,EAAE,CACF,eAAe,CAGb,SAAS,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;IAEhE,sCAAsC;IACtC,GAAG,EAAE,CAIH,OAAa,EACb,SAAiB,EAKjB,EAAE,CACF,eAAe,CAGb,OAAO,OAAO,CAAC,aAAa,KAAK,SAAS,CAAC,aAAa,GAAG,CAAC;IAEhE,uCAAuC;IACvC,IAAI,EAAE,GAAqB,EAAE,CAAC,eAAe,CAAiB,MAAM,CAAC;IAErE,uCAAuC;IAEvC,yCAAyC;IACzC,KAAK,EAAE,CAAoC,GAAG,MAAe,EAAE,EAAE;QAC/D,MAAM,WAAW,GAAG,MAAM;aACvB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;aACvD,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,OAAO,eAAe,CACpB,SAAS,WAAW,GAAyB,CAC9C,CAAC;IACJ,CAAC;IAED,2CAA2C;IAC3C,MAAM,EAAE,CAAoC,GAAG,MAAe,EAAE,EAAE;QAChE,MAAM,WAAW,GAAG,MAAM;aACvB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;aACvD,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,OAAO,eAAe,CACpB,UAAU,WAAW,GAA0B,CAChD,CAAC;IACJ,CAAC;IAED,0CAA0C;IAE1C,0BAA0B;IAC1B,IAAI,EAAE,GAAG,EAAE,CAAC,eAAe,CAAiB,MAAM,CAAC;IAEnD,0BAA0B;IAC1B,IAAI,EAAE,GAAG,EAAE,CAAC,eAAe,CAAiB,MAAM,CAAC;IAEnD,sDAAsD;IAEtD,8EAA8E;IAC9E,uBAAuB,EAAE,CAIvB,IAAW,EACX,IAAW,EAKX,EAAE,CACF,eAAe,CAGb,2BAA2B,IAAI,KAAK,IAAI,CAAC,aAAa,GAAG,CAAC;IAE9D,yEAAyE;IACzE,iBAAiB,EAAE,CAIjB,IAAW,EACX,IAAW,EAKX,EAAE,CACF,eAAe,CAGb,qBAAqB,IAAI,KAAK,IAAI,CAAC,aAAa,GAAG,CAAC;CAChD,CAAC;AAWX,qDAAqD;AACrD,MAAM,UAAU,eAAe,CAAC,KAAc;IAC5C,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,eAAe,IAAI,KAAK;QACvB,KAAiC,CAAC,eAAe,CAAC,KAAK,IAAI,CAC7D,CAAC;AACJ,CAAC;AAED,oDAAoD;AACpD,MAAM,UAAU,eAAe,CAAC,SAA2B;IACzD,OAAO,SAAS,CAAC,aAAa,CAAC;AACjC,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,YAAY,CAAC,SAA2B;IACtD,OAAO,SAAS,CAAC,UAAU,CAAC;AAC9B,CAAC"}
|
package/package.json
CHANGED
package/src/api/api.test.ts
CHANGED
|
@@ -116,7 +116,7 @@ describe("TinybirdApi", () => {
|
|
|
116
116
|
});
|
|
117
117
|
|
|
118
118
|
const result = await api.query<{ pathname: string; views: number }>("top_pages", {
|
|
119
|
-
start_date:
|
|
119
|
+
start_date: "2024-01-01 00:00:00",
|
|
120
120
|
limit: 5,
|
|
121
121
|
tags: ["a", "b"],
|
|
122
122
|
});
|
|
@@ -124,11 +124,24 @@ describe("TinybirdApi", () => {
|
|
|
124
124
|
expect(result.rows).toBe(1);
|
|
125
125
|
expect(result.data[0]).toEqual({ pathname: "/", views: 1 });
|
|
126
126
|
expect(fromParam).toBe(TINYBIRD_FROM_PARAM);
|
|
127
|
-
expect(startDateParam).toBe("2024-01-
|
|
127
|
+
expect(startDateParam).toBe("2024-01-01 00:00:00");
|
|
128
128
|
expect(limitParam).toBe("5");
|
|
129
129
|
expect(tagsParams).toEqual(["a", "b"]);
|
|
130
130
|
});
|
|
131
131
|
|
|
132
|
+
it("throws when query params include Date values", async () => {
|
|
133
|
+
const api = createTinybirdApi({
|
|
134
|
+
baseUrl: BASE_URL,
|
|
135
|
+
token: "p.default-token",
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
await expect(
|
|
139
|
+
api.query("top_pages", {
|
|
140
|
+
start_date: new Date("2024-01-01T00:00:00.000Z"),
|
|
141
|
+
})
|
|
142
|
+
).rejects.toThrow("Date values are not supported for query parameter");
|
|
143
|
+
});
|
|
144
|
+
|
|
132
145
|
it("ingests rows via tinybirdApi.ingest", async () => {
|
|
133
146
|
let datasourceName: string | null = null;
|
|
134
147
|
let waitParam: string | null = null;
|
|
@@ -160,11 +173,11 @@ describe("TinybirdApi", () => {
|
|
|
160
173
|
});
|
|
161
174
|
|
|
162
175
|
const result = await api.ingest("events", {
|
|
163
|
-
timestamp:
|
|
176
|
+
timestamp: "2024-01-01 00:00:00",
|
|
164
177
|
count: 10n,
|
|
165
178
|
payload: new Map([["k", "v"]]),
|
|
166
179
|
nested: {
|
|
167
|
-
when:
|
|
180
|
+
when: "2024-01-02 00:00:00",
|
|
168
181
|
},
|
|
169
182
|
});
|
|
170
183
|
|
|
@@ -174,13 +187,26 @@ describe("TinybirdApi", () => {
|
|
|
174
187
|
expect(fromParam).toBe(TINYBIRD_FROM_PARAM);
|
|
175
188
|
expect(contentType).toBe("application/x-ndjson");
|
|
176
189
|
expect(parsedBody).toEqual({
|
|
177
|
-
timestamp: "2024-01-
|
|
190
|
+
timestamp: "2024-01-01 00:00:00",
|
|
178
191
|
count: "10",
|
|
179
192
|
payload: { k: "v" },
|
|
180
|
-
nested: { when: "2024-01-
|
|
193
|
+
nested: { when: "2024-01-02 00:00:00" },
|
|
181
194
|
});
|
|
182
195
|
});
|
|
183
196
|
|
|
197
|
+
it("throws when ingest payload includes Date values", async () => {
|
|
198
|
+
const api = createTinybirdApi({
|
|
199
|
+
baseUrl: BASE_URL,
|
|
200
|
+
token: "p.default-token",
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
await expect(
|
|
204
|
+
api.ingest("events", {
|
|
205
|
+
timestamp: new Date("2024-01-01T00:00:00.000Z"),
|
|
206
|
+
})
|
|
207
|
+
).rejects.toThrow("Date values are not supported in ingest payloads");
|
|
208
|
+
});
|
|
209
|
+
|
|
184
210
|
it("executes raw SQL via tinybirdApi.sql", async () => {
|
|
185
211
|
let rawSql: string | null = null;
|
|
186
212
|
let contentType: string | null = null;
|
package/src/api/api.ts
CHANGED
|
@@ -212,14 +212,22 @@ export class TinybirdApi {
|
|
|
212
212
|
|
|
213
213
|
if (Array.isArray(value)) {
|
|
214
214
|
for (const item of value) {
|
|
215
|
+
if (item instanceof Date) {
|
|
216
|
+
throw new Error(
|
|
217
|
+
`Date values are not supported for query parameter "${key}". ` +
|
|
218
|
+
"Pass a string in YYYY-MM-DD HH:MM:SS format (or YYYY-MM-DD HH:MM:SS.sss for DateTime64)."
|
|
219
|
+
);
|
|
220
|
+
}
|
|
215
221
|
url.searchParams.append(key, String(item));
|
|
216
222
|
}
|
|
217
223
|
continue;
|
|
218
224
|
}
|
|
219
225
|
|
|
220
226
|
if (value instanceof Date) {
|
|
221
|
-
|
|
222
|
-
|
|
227
|
+
throw new Error(
|
|
228
|
+
`Date values are not supported for query parameter "${key}". ` +
|
|
229
|
+
"Pass a string in YYYY-MM-DD HH:MM:SS format (or YYYY-MM-DD HH:MM:SS.sss for DateTime64)."
|
|
230
|
+
);
|
|
223
231
|
}
|
|
224
232
|
|
|
225
233
|
url.searchParams.set(key, String(value));
|
|
@@ -581,7 +589,10 @@ export class TinybirdApi {
|
|
|
581
589
|
|
|
582
590
|
private serializeValue(value: unknown): unknown {
|
|
583
591
|
if (value instanceof Date) {
|
|
584
|
-
|
|
592
|
+
throw new Error(
|
|
593
|
+
"Date values are not supported in ingest payloads. " +
|
|
594
|
+
"Pass strings in YYYY-MM-DD, YYYY-MM-DD HH:MM:SS, or YYYY-MM-DD HH:MM:SS.sss format."
|
|
595
|
+
);
|
|
585
596
|
}
|
|
586
597
|
|
|
587
598
|
if (value instanceof Map) {
|
|
@@ -15,11 +15,11 @@ const EXPECTED_COMPLEX_OUTPUT = `/**
|
|
|
15
15
|
* Review endpoint output schemas and any defaults before production use.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import {
|
|
18
|
+
import { defineKafkaConnection, defineDatasource, definePipe, defineMaterializedView, defineCopyPipe, node, t, engine, column, p } from "@tinybirdco/sdk";
|
|
19
19
|
|
|
20
20
|
// Connections
|
|
21
21
|
|
|
22
|
-
export const stream =
|
|
22
|
+
export const stream = defineKafkaConnection("stream", {
|
|
23
23
|
bootstrapServers: "localhost:9092",
|
|
24
24
|
securityProtocol: "SASL_SSL",
|
|
25
25
|
saslMechanism: "PLAIN",
|
|
@@ -183,11 +183,11 @@ const EXPECTED_PARTIAL_OUTPUT = `/**
|
|
|
183
183
|
* Review endpoint output schemas and any defaults before production use.
|
|
184
184
|
*/
|
|
185
185
|
|
|
186
|
-
import {
|
|
186
|
+
import { defineKafkaConnection, defineDatasource, definePipe, defineMaterializedView, defineCopyPipe, node, t, engine, p } from "@tinybirdco/sdk";
|
|
187
187
|
|
|
188
188
|
// Connections
|
|
189
189
|
|
|
190
|
-
export const stream =
|
|
190
|
+
export const stream = defineKafkaConnection("stream", {
|
|
191
191
|
bootstrapServers: "localhost:9092",
|
|
192
192
|
});
|
|
193
193
|
|
|
@@ -561,4 +561,58 @@ TOKEN endpoint_token READ
|
|
|
561
561
|
expect(result.outputContent).toBe(EXPECTED_PARTIAL_OUTPUT);
|
|
562
562
|
expect(fs.existsSync(result.outputPath)).toBe(false);
|
|
563
563
|
});
|
|
564
|
+
|
|
565
|
+
it("migrates s3 connection and import datasource directives", async () => {
|
|
566
|
+
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
|
|
567
|
+
tempDirs.push(tempDir);
|
|
568
|
+
|
|
569
|
+
writeFile(
|
|
570
|
+
tempDir,
|
|
571
|
+
"s3sample.connection",
|
|
572
|
+
`TYPE s3
|
|
573
|
+
S3_REGION "us-east-1"
|
|
574
|
+
S3_ARN "arn:aws:iam::123456789012:role/tinybird-s3-access"
|
|
575
|
+
`
|
|
576
|
+
);
|
|
577
|
+
|
|
578
|
+
writeFile(
|
|
579
|
+
tempDir,
|
|
580
|
+
"events_landing.datasource",
|
|
581
|
+
`SCHEMA >
|
|
582
|
+
timestamp DateTime,
|
|
583
|
+
session_id String
|
|
584
|
+
|
|
585
|
+
ENGINE "MergeTree"
|
|
586
|
+
ENGINE_SORTING_KEY "timestamp"
|
|
587
|
+
IMPORT_CONNECTION_NAME s3sample
|
|
588
|
+
IMPORT_BUCKET_URI s3://my-bucket/events/*.csv
|
|
589
|
+
IMPORT_SCHEDULE @auto
|
|
590
|
+
IMPORT_FROM_TIMESTAMP 2024-01-01T00:00:00Z
|
|
591
|
+
`
|
|
592
|
+
);
|
|
593
|
+
|
|
594
|
+
const result = await runMigrate({
|
|
595
|
+
cwd: tempDir,
|
|
596
|
+
patterns: ["."],
|
|
597
|
+
strict: true,
|
|
598
|
+
});
|
|
599
|
+
|
|
600
|
+
expect(result.success).toBe(true);
|
|
601
|
+
expect(result.errors).toHaveLength(0);
|
|
602
|
+
expect(result.migrated.filter((resource) => resource.kind === "connection")).toHaveLength(1);
|
|
603
|
+
expect(result.migrated.filter((resource) => resource.kind === "datasource")).toHaveLength(1);
|
|
604
|
+
|
|
605
|
+
const output = fs.readFileSync(result.outputPath, "utf-8");
|
|
606
|
+
expect(output).toContain("defineS3Connection");
|
|
607
|
+
expect(output).toContain('export const s3sample = defineS3Connection("s3sample", {');
|
|
608
|
+
expect(output).toContain('region: "us-east-1"');
|
|
609
|
+
expect(output).toContain(
|
|
610
|
+
'arn: "arn:aws:iam::123456789012:role/tinybird-s3-access"'
|
|
611
|
+
);
|
|
612
|
+
expect(output).toContain("s3: {");
|
|
613
|
+
expect(output).toContain("connection: s3sample");
|
|
614
|
+
expect(output).toContain('bucketUri: "s3://my-bucket/events/*.csv"');
|
|
615
|
+
expect(output).toContain('schedule: "@auto"');
|
|
616
|
+
expect(output).toContain('fromTimestamp: "2024-01-01T00:00:00Z"');
|
|
617
|
+
});
|
|
564
618
|
});
|
|
@@ -129,15 +129,18 @@ export async function runMigrate(
|
|
|
129
129
|
}
|
|
130
130
|
|
|
131
131
|
for (const datasource of parsedDatasources) {
|
|
132
|
+
const referencedConnectionName =
|
|
133
|
+
datasource.kafka?.connectionName ?? datasource.s3?.connectionName;
|
|
134
|
+
|
|
132
135
|
if (
|
|
133
|
-
|
|
134
|
-
!migratedConnectionNames.has(
|
|
136
|
+
referencedConnectionName &&
|
|
137
|
+
!migratedConnectionNames.has(referencedConnectionName)
|
|
135
138
|
) {
|
|
136
139
|
errors.push({
|
|
137
140
|
filePath: datasource.filePath,
|
|
138
141
|
resourceName: datasource.name,
|
|
139
142
|
resourceKind: datasource.kind,
|
|
140
|
-
message: `Datasource references missing/unmigrated connection "${
|
|
143
|
+
message: `Datasource references missing/unmigrated connection "${referencedConnectionName}".`,
|
|
141
144
|
});
|
|
142
145
|
continue;
|
|
143
146
|
}
|
|
@@ -237,4 +240,3 @@ export async function runMigrate(
|
|
|
237
240
|
outputContent,
|
|
238
241
|
};
|
|
239
242
|
}
|
|
240
|
-
|
package/src/client/base.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { describe, it, expect } from "vitest";
|
|
2
2
|
import { generateConnection, generateAllConnections } from "./connection.js";
|
|
3
|
-
import {
|
|
3
|
+
import { defineKafkaConnection, defineS3Connection } from "../schema/connection.js";
|
|
4
4
|
|
|
5
5
|
describe("Connection Generator", () => {
|
|
6
6
|
describe("generateConnection", () => {
|
|
7
7
|
it("generates basic Kafka connection with required fields", () => {
|
|
8
|
-
const conn =
|
|
8
|
+
const conn = defineKafkaConnection("my_kafka", {
|
|
9
9
|
bootstrapServers: "kafka.example.com:9092",
|
|
10
10
|
});
|
|
11
11
|
|
|
@@ -17,7 +17,7 @@ describe("Connection Generator", () => {
|
|
|
17
17
|
});
|
|
18
18
|
|
|
19
19
|
it("includes security protocol when provided", () => {
|
|
20
|
-
const conn =
|
|
20
|
+
const conn = defineKafkaConnection("my_kafka", {
|
|
21
21
|
bootstrapServers: "kafka.example.com:9092",
|
|
22
22
|
securityProtocol: "SASL_SSL",
|
|
23
23
|
});
|
|
@@ -28,7 +28,7 @@ describe("Connection Generator", () => {
|
|
|
28
28
|
});
|
|
29
29
|
|
|
30
30
|
it("includes SASL mechanism when provided", () => {
|
|
31
|
-
const conn =
|
|
31
|
+
const conn = defineKafkaConnection("my_kafka", {
|
|
32
32
|
bootstrapServers: "kafka.example.com:9092",
|
|
33
33
|
saslMechanism: "PLAIN",
|
|
34
34
|
});
|
|
@@ -39,7 +39,7 @@ describe("Connection Generator", () => {
|
|
|
39
39
|
});
|
|
40
40
|
|
|
41
41
|
it("includes key and secret when provided", () => {
|
|
42
|
-
const conn =
|
|
42
|
+
const conn = defineKafkaConnection("my_kafka", {
|
|
43
43
|
bootstrapServers: "kafka.example.com:9092",
|
|
44
44
|
key: '{{ tb_secret("KAFKA_KEY") }}',
|
|
45
45
|
secret: '{{ tb_secret("KAFKA_SECRET") }}',
|
|
@@ -52,7 +52,7 @@ describe("Connection Generator", () => {
|
|
|
52
52
|
});
|
|
53
53
|
|
|
54
54
|
it("includes SSL CA PEM when provided", () => {
|
|
55
|
-
const conn =
|
|
55
|
+
const conn = defineKafkaConnection("my_kafka", {
|
|
56
56
|
bootstrapServers: "kafka.example.com:9092",
|
|
57
57
|
sslCaPem: '{{ tb_secret("KAFKA_CA_CERT") }}',
|
|
58
58
|
});
|
|
@@ -63,7 +63,7 @@ describe("Connection Generator", () => {
|
|
|
63
63
|
});
|
|
64
64
|
|
|
65
65
|
it("generates full Kafka connection with all options", () => {
|
|
66
|
-
const conn =
|
|
66
|
+
const conn = defineKafkaConnection("my_kafka", {
|
|
67
67
|
bootstrapServers: "kafka.example.com:9092",
|
|
68
68
|
securityProtocol: "SASL_SSL",
|
|
69
69
|
saslMechanism: "SCRAM-SHA-256",
|
|
@@ -85,7 +85,7 @@ describe("Connection Generator", () => {
|
|
|
85
85
|
});
|
|
86
86
|
|
|
87
87
|
it("supports PLAINTEXT security protocol", () => {
|
|
88
|
-
const conn =
|
|
88
|
+
const conn = defineKafkaConnection("local_kafka", {
|
|
89
89
|
bootstrapServers: "localhost:9092",
|
|
90
90
|
securityProtocol: "PLAINTEXT",
|
|
91
91
|
});
|
|
@@ -99,7 +99,7 @@ describe("Connection Generator", () => {
|
|
|
99
99
|
const mechanisms = ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512", "OAUTHBEARER"] as const;
|
|
100
100
|
|
|
101
101
|
mechanisms.forEach((mechanism) => {
|
|
102
|
-
const conn =
|
|
102
|
+
const conn = defineKafkaConnection("my_kafka", {
|
|
103
103
|
bootstrapServers: "kafka.example.com:9092",
|
|
104
104
|
saslMechanism: mechanism,
|
|
105
105
|
});
|
|
@@ -109,21 +109,52 @@ describe("Connection Generator", () => {
|
|
|
109
109
|
expect(result.content).toContain(`KAFKA_SASL_MECHANISM ${mechanism}`);
|
|
110
110
|
});
|
|
111
111
|
});
|
|
112
|
+
|
|
113
|
+
it("generates basic S3 connection with IAM role auth", () => {
|
|
114
|
+
const conn = defineS3Connection("my_s3", {
|
|
115
|
+
region: "us-east-1",
|
|
116
|
+
arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
const result = generateConnection(conn);
|
|
120
|
+
|
|
121
|
+
expect(result.name).toBe("my_s3");
|
|
122
|
+
expect(result.content).toContain("TYPE s3");
|
|
123
|
+
expect(result.content).toContain("S3_REGION us-east-1");
|
|
124
|
+
expect(result.content).toContain(
|
|
125
|
+
"S3_ARN arn:aws:iam::123456789012:role/tinybird-s3-access"
|
|
126
|
+
);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
it("generates S3 connection with access key auth", () => {
|
|
130
|
+
const conn = defineS3Connection("my_s3", {
|
|
131
|
+
region: "us-east-1",
|
|
132
|
+
accessKey: '{{ tb_secret("S3_ACCESS_KEY") }}',
|
|
133
|
+
secret: '{{ tb_secret("S3_SECRET") }}',
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
const result = generateConnection(conn);
|
|
137
|
+
|
|
138
|
+
expect(result.content).toContain("TYPE s3");
|
|
139
|
+
expect(result.content).toContain('S3_ACCESS_KEY {{ tb_secret("S3_ACCESS_KEY") }}');
|
|
140
|
+
expect(result.content).toContain('S3_SECRET {{ tb_secret("S3_SECRET") }}');
|
|
141
|
+
});
|
|
112
142
|
});
|
|
113
143
|
|
|
114
144
|
describe("generateAllConnections", () => {
|
|
115
145
|
it("generates all connections", () => {
|
|
116
|
-
const conn1 =
|
|
146
|
+
const conn1 = defineKafkaConnection("kafka1", {
|
|
117
147
|
bootstrapServers: "kafka1.example.com:9092",
|
|
118
148
|
});
|
|
119
|
-
const conn2 =
|
|
120
|
-
|
|
149
|
+
const conn2 = defineS3Connection("s3_logs", {
|
|
150
|
+
region: "us-east-1",
|
|
151
|
+
arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
|
|
121
152
|
});
|
|
122
153
|
|
|
123
|
-
const results = generateAllConnections({ kafka1: conn1,
|
|
154
|
+
const results = generateAllConnections({ kafka1: conn1, s3_logs: conn2 });
|
|
124
155
|
|
|
125
156
|
expect(results).toHaveLength(2);
|
|
126
|
-
expect(results.map((r) => r.name).sort()).toEqual(["kafka1", "
|
|
157
|
+
expect(results.map((r) => r.name).sort()).toEqual(["kafka1", "s3_logs"]);
|
|
127
158
|
});
|
|
128
159
|
|
|
129
160
|
it("returns empty array for empty connections", () => {
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import type { ConnectionDefinition, KafkaConnectionDefinition } from "../schema/connection.js";
|
|
7
|
+
import { isS3ConnectionDefinition, type S3ConnectionDefinition } from "../schema/connection.js";
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
10
|
* Generated connection content
|
|
@@ -48,6 +49,31 @@ function generateKafkaConnection(connection: KafkaConnectionDefinition): string
|
|
|
48
49
|
return parts.join("\n");
|
|
49
50
|
}
|
|
50
51
|
|
|
52
|
+
/**
|
|
53
|
+
* Generate an S3 connection content
|
|
54
|
+
*/
|
|
55
|
+
function generateS3Connection(connection: S3ConnectionDefinition): string {
|
|
56
|
+
const parts: string[] = [];
|
|
57
|
+
const options = connection.options;
|
|
58
|
+
|
|
59
|
+
parts.push("TYPE s3");
|
|
60
|
+
parts.push(`S3_REGION ${options.region}`);
|
|
61
|
+
|
|
62
|
+
if (options.arn) {
|
|
63
|
+
parts.push(`S3_ARN ${options.arn}`);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if (options.accessKey) {
|
|
67
|
+
parts.push(`S3_ACCESS_KEY ${options.accessKey}`);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (options.secret) {
|
|
71
|
+
parts.push(`S3_SECRET ${options.secret}`);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return parts.join("\n");
|
|
75
|
+
}
|
|
76
|
+
|
|
51
77
|
/**
|
|
52
78
|
* Generate a .connection file content from a ConnectionDefinition
|
|
53
79
|
*
|
|
@@ -56,7 +82,7 @@ function generateKafkaConnection(connection: KafkaConnectionDefinition): string
|
|
|
56
82
|
*
|
|
57
83
|
* @example
|
|
58
84
|
* ```ts
|
|
59
|
-
* const myKafka =
|
|
85
|
+
* const myKafka = defineKafkaConnection('my_kafka', {
|
|
60
86
|
* bootstrapServers: 'kafka.example.com:9092',
|
|
61
87
|
* securityProtocol: 'SASL_SSL',
|
|
62
88
|
* saslMechanism: 'PLAIN',
|
|
@@ -81,8 +107,10 @@ export function generateConnection(
|
|
|
81
107
|
|
|
82
108
|
if (connection._connectionType === "kafka") {
|
|
83
109
|
content = generateKafkaConnection(connection as KafkaConnectionDefinition);
|
|
110
|
+
} else if (isS3ConnectionDefinition(connection)) {
|
|
111
|
+
content = generateS3Connection(connection);
|
|
84
112
|
} else {
|
|
85
|
-
throw new Error(
|
|
113
|
+
throw new Error("Unsupported connection type.");
|
|
86
114
|
}
|
|
87
115
|
|
|
88
116
|
return {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest';
|
|
2
2
|
import { generateDatasource, generateAllDatasources } from './datasource.js';
|
|
3
3
|
import { defineDatasource } from '../schema/datasource.js';
|
|
4
|
-
import {
|
|
4
|
+
import { defineKafkaConnection, defineS3Connection } from '../schema/connection.js';
|
|
5
5
|
import { defineToken } from '../schema/token.js';
|
|
6
6
|
import { t } from '../schema/types.js';
|
|
7
7
|
import { engine } from '../schema/engines.js';
|
|
@@ -180,10 +180,10 @@ describe('Datasource Generator', () => {
|
|
|
180
180
|
expect(result.content).toContain('is_deleted Bool `json:$.is_deleted` DEFAULT 0');
|
|
181
181
|
});
|
|
182
182
|
|
|
183
|
-
it('formats
|
|
183
|
+
it('formats string default values for DateTime type', () => {
|
|
184
184
|
const ds = defineDatasource('test_ds', {
|
|
185
185
|
schema: {
|
|
186
|
-
created_at: t.dateTime().default(
|
|
186
|
+
created_at: t.dateTime().default('2024-01-15 10:30:00'),
|
|
187
187
|
},
|
|
188
188
|
});
|
|
189
189
|
|
|
@@ -191,10 +191,10 @@ describe('Datasource Generator', () => {
|
|
|
191
191
|
expect(result.content).toContain("created_at DateTime `json:$.created_at` DEFAULT '2024-01-15 10:30:00'");
|
|
192
192
|
});
|
|
193
193
|
|
|
194
|
-
it('formats
|
|
194
|
+
it('formats string default values for Date type', () => {
|
|
195
195
|
const ds = defineDatasource('test_ds', {
|
|
196
196
|
schema: {
|
|
197
|
-
birth_date: t.date().default(
|
|
197
|
+
birth_date: t.date().default('2024-01-15'),
|
|
198
198
|
},
|
|
199
199
|
});
|
|
200
200
|
|
|
@@ -312,7 +312,7 @@ describe('Datasource Generator', () => {
|
|
|
312
312
|
|
|
313
313
|
describe('Kafka configuration', () => {
|
|
314
314
|
it('includes Kafka connection name and topic', () => {
|
|
315
|
-
const kafkaConn =
|
|
315
|
+
const kafkaConn = defineKafkaConnection('my_kafka', {
|
|
316
316
|
bootstrapServers: 'kafka.example.com:9092',
|
|
317
317
|
});
|
|
318
318
|
|
|
@@ -335,7 +335,7 @@ describe('Datasource Generator', () => {
|
|
|
335
335
|
});
|
|
336
336
|
|
|
337
337
|
it('includes Kafka group ID when provided', () => {
|
|
338
|
-
const kafkaConn =
|
|
338
|
+
const kafkaConn = defineKafkaConnection('my_kafka', {
|
|
339
339
|
bootstrapServers: 'kafka.example.com:9092',
|
|
340
340
|
});
|
|
341
341
|
|
|
@@ -358,7 +358,7 @@ describe('Datasource Generator', () => {
|
|
|
358
358
|
});
|
|
359
359
|
|
|
360
360
|
it('includes auto offset reset when provided', () => {
|
|
361
|
-
const kafkaConn =
|
|
361
|
+
const kafkaConn = defineKafkaConnection('my_kafka', {
|
|
362
362
|
bootstrapServers: 'kafka.example.com:9092',
|
|
363
363
|
});
|
|
364
364
|
|
|
@@ -381,7 +381,7 @@ describe('Datasource Generator', () => {
|
|
|
381
381
|
});
|
|
382
382
|
|
|
383
383
|
it('generates complete Kafka datasource with all options', () => {
|
|
384
|
-
const kafkaConn =
|
|
384
|
+
const kafkaConn = defineKafkaConnection('my_kafka', {
|
|
385
385
|
bootstrapServers: 'kafka.example.com:9092',
|
|
386
386
|
securityProtocol: 'SASL_SSL',
|
|
387
387
|
saslMechanism: 'PLAIN',
|
|
@@ -417,6 +417,58 @@ describe('Datasource Generator', () => {
|
|
|
417
417
|
});
|
|
418
418
|
});
|
|
419
419
|
|
|
420
|
+
describe('S3 configuration', () => {
|
|
421
|
+
it('includes S3 connection name and bucket uri', () => {
|
|
422
|
+
const s3Conn = defineS3Connection('my_s3', {
|
|
423
|
+
region: 'us-east-1',
|
|
424
|
+
arn: 'arn:aws:iam::123456789012:role/tinybird-s3-access',
|
|
425
|
+
});
|
|
426
|
+
|
|
427
|
+
const ds = defineDatasource('s3_events', {
|
|
428
|
+
schema: {
|
|
429
|
+
timestamp: t.dateTime(),
|
|
430
|
+
event: t.string(),
|
|
431
|
+
},
|
|
432
|
+
engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
|
|
433
|
+
s3: {
|
|
434
|
+
connection: s3Conn,
|
|
435
|
+
bucketUri: 's3://my-bucket/events/*.csv',
|
|
436
|
+
},
|
|
437
|
+
});
|
|
438
|
+
|
|
439
|
+
const result = generateDatasource(ds);
|
|
440
|
+
|
|
441
|
+
expect(result.content).toContain('IMPORT_CONNECTION_NAME my_s3');
|
|
442
|
+
expect(result.content).toContain('IMPORT_BUCKET_URI s3://my-bucket/events/*.csv');
|
|
443
|
+
});
|
|
444
|
+
|
|
445
|
+
it('includes optional S3 schedule and from timestamp', () => {
|
|
446
|
+
const s3Conn = defineS3Connection('my_s3', {
|
|
447
|
+
region: 'us-east-1',
|
|
448
|
+
arn: 'arn:aws:iam::123456789012:role/tinybird-s3-access',
|
|
449
|
+
});
|
|
450
|
+
|
|
451
|
+
const ds = defineDatasource('s3_events', {
|
|
452
|
+
schema: {
|
|
453
|
+
timestamp: t.dateTime(),
|
|
454
|
+
event: t.string(),
|
|
455
|
+
},
|
|
456
|
+
engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
|
|
457
|
+
s3: {
|
|
458
|
+
connection: s3Conn,
|
|
459
|
+
bucketUri: 's3://my-bucket/events/*.csv',
|
|
460
|
+
schedule: '@auto',
|
|
461
|
+
fromTimestamp: '2024-01-01T00:00:00Z',
|
|
462
|
+
},
|
|
463
|
+
});
|
|
464
|
+
|
|
465
|
+
const result = generateDatasource(ds);
|
|
466
|
+
|
|
467
|
+
expect(result.content).toContain('IMPORT_SCHEDULE @auto');
|
|
468
|
+
expect(result.content).toContain('IMPORT_FROM_TIMESTAMP 2024-01-01T00:00:00Z');
|
|
469
|
+
});
|
|
470
|
+
});
|
|
471
|
+
|
|
420
472
|
describe('Token generation', () => {
|
|
421
473
|
it('generates TOKEN lines with inline config', () => {
|
|
422
474
|
const ds = defineDatasource('test_ds', {
|
|
@@ -3,7 +3,14 @@
|
|
|
3
3
|
* Converts DatasourceDefinition to native .datasource file format
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import type {
|
|
6
|
+
import type {
|
|
7
|
+
DatasourceDefinition,
|
|
8
|
+
SchemaDefinition,
|
|
9
|
+
ColumnDefinition,
|
|
10
|
+
KafkaConfig,
|
|
11
|
+
S3Config,
|
|
12
|
+
TokenConfig,
|
|
13
|
+
} from "../schema/datasource.js";
|
|
7
14
|
import type { AnyTypeValidator, TypeModifiers } from "../schema/types.js";
|
|
8
15
|
import { getColumnType, getColumnJsonPath } from "../schema/datasource.js";
|
|
9
16
|
import { getEngineClause, type EngineConfig } from "../schema/engines.js";
|
|
@@ -163,6 +170,26 @@ function generateKafkaConfig(kafka: KafkaConfig): string {
|
|
|
163
170
|
return parts.join("\n");
|
|
164
171
|
}
|
|
165
172
|
|
|
173
|
+
/**
|
|
174
|
+
* Generate S3 import configuration lines
|
|
175
|
+
*/
|
|
176
|
+
function generateS3Config(s3: S3Config): string {
|
|
177
|
+
const parts: string[] = [];
|
|
178
|
+
|
|
179
|
+
parts.push(`IMPORT_CONNECTION_NAME ${s3.connection._name}`);
|
|
180
|
+
parts.push(`IMPORT_BUCKET_URI ${s3.bucketUri}`);
|
|
181
|
+
|
|
182
|
+
if (s3.schedule) {
|
|
183
|
+
parts.push(`IMPORT_SCHEDULE ${s3.schedule}`);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
if (s3.fromTimestamp) {
|
|
187
|
+
parts.push(`IMPORT_FROM_TIMESTAMP ${s3.fromTimestamp}`);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
return parts.join("\n");
|
|
191
|
+
}
|
|
192
|
+
|
|
166
193
|
/**
|
|
167
194
|
* Generate forward query section
|
|
168
195
|
*/
|
|
@@ -261,6 +288,10 @@ export function generateDatasource(
|
|
|
261
288
|
): GeneratedDatasource {
|
|
262
289
|
const parts: string[] = [];
|
|
263
290
|
|
|
291
|
+
if (datasource.options.kafka && datasource.options.s3) {
|
|
292
|
+
throw new Error("Datasource cannot define both `kafka` and `s3` ingestion options.");
|
|
293
|
+
}
|
|
294
|
+
|
|
264
295
|
// Add description if present
|
|
265
296
|
if (datasource.options.description) {
|
|
266
297
|
parts.push(`DESCRIPTION >\n ${datasource.options.description}`);
|
|
@@ -283,6 +314,12 @@ export function generateDatasource(
|
|
|
283
314
|
parts.push(generateKafkaConfig(datasource.options.kafka));
|
|
284
315
|
}
|
|
285
316
|
|
|
317
|
+
// Add S3 configuration if present
|
|
318
|
+
if (datasource.options.s3) {
|
|
319
|
+
parts.push("");
|
|
320
|
+
parts.push(generateS3Config(datasource.options.s3));
|
|
321
|
+
}
|
|
322
|
+
|
|
286
323
|
// Add forward query if present
|
|
287
324
|
const forwardQuery = generateForwardQuery(datasource.options.forwardQuery);
|
|
288
325
|
if (forwardQuery) {
|