@sasjs/utils 2.48.2 → 2.49.1

package/macros/mp_ds2cards.sas

@@ -0,0 +1,291 @@
+/**
+  @file
+  @brief Create a CARDS file from a SAS dataset.
+  @details Uses dataset attributes to convert all data into datalines.
+    Running the generated file will rebuild the original dataset.  Includes
+    support for large decimals, binary data, PROCESSED_DTTM columns, and
+    alternative encoding.  If the input dataset is empty, the cards file will
+    still be created.
+
+    Additional support to generate a random sample and max rows.
+
+  Usage:
+
+      %mp_ds2cards(sashelp.class
+        , tgt_ds=work.class
+        , cards_file= "C:\temp\class.sas"
+        , showlog=NO
+        , maxobs=5
+      )
+
+  TODO:
+    - labelling the dataset
+    - explicity setting a unix LF
+    - constraints / indexes etc
+
+  @param [in] base_ds Should be two level - eg work.blah.  This is the table
+                  that is converted to a cards file.
+  @param [in] tgt_ds= Table that the generated cards file would create.
+    Optional - if omitted, will be same as BASE_DS.
+  @param [out] cards_file= ("%sysfunc(pathname(work))/cardgen.sas") Location in
+    which to write the (.sas) cards file
+  @param [in] maxobs= (max) To limit output to the first <code>maxobs</code>
+    observations, enter an integer here.
+  @param [in] random_sample= (NO) Set to YES to generate a random sample of
+    data.  Can be quite slow.
+  @param [in] showlog= (YES) Whether to show generated cards file in the SAS
+    log.  Valid values:
+    @li YES
+    @li NO
+  @param [in] outencoding= Provide encoding value for file statement (eg utf-8)
+  @param [in] append= (NO) If NO then will rebuild the cards file if it already
+    exists, otherwise will append to it.  Used by the mp_lib2cards.sas macro.
+
+  <h4> Related Macros </h4>
+  @li mp_lib2cards.sas
+  @li mp_ds2inserts.sas
+  @li mp_mdtablewrite.sas
+
+  @version 9.2
+  @author Allan Bowe
+  @cond
+**/
+
+%macro mp_ds2cards(base_ds, tgt_ds=
+    ,cards_file="%sysfunc(pathname(work))/cardgen.sas"
+    ,maxobs=max
+    ,random_sample=NO
+    ,showlog=YES
+    ,outencoding=
+    ,append=NO
+)/*/STORE SOURCE*/;
+%local i setds nvars;
+
+%if not %sysfunc(exist(&base_ds)) %then %do;
+  %put %str(WARN)ING:  &base_ds does not exist;
+  %return;
+%end;
+
+%if %index(&base_ds,.)=0 %then %let base_ds=WORK.&base_ds;
+%if (&tgt_ds = ) %then %let tgt_ds=&base_ds;
+%if %index(&tgt_ds,.)=0 %then %let tgt_ds=WORK.%scan(&base_ds,2,.);
+%if ("&outencoding" ne "") %then %let outencoding=encoding="&outencoding";
+%if ("&append" = "" or "&append" = "NO") %then %let append=;
+%else %let append=mod;
+
+/* get varcount */
+%let nvars=0;
+proc sql noprint;
+select count(*) into: nvars from dictionary.columns
+  where upcase(libname)="%scan(%upcase(&base_ds),1)"
+    and upcase(memname)="%scan(%upcase(&base_ds),2)";
+%if &nvars=0 %then %do;
+  %put %str(WARN)ING: Dataset &base_ds has no variables, will not be converted.;
+  %return;
+%end;
+
+/* get indexes */
+proc sort
+  data=sashelp.vindex(
+    where=(upcase(libname)="%scan(%upcase(&base_ds),1)"
+      and upcase(memname)="%scan(%upcase(&base_ds),2)")
+    )
+  out=_data_;
+  by indxname indxpos;
+run;
+
+%local indexes;
+data _null_;
+  set &syslast end=last;
+  if _n_=1 then call symputx('indexes','(index=(','l');
+  by indxname indxpos;
+  length vars $32767 nom uni $8;
+  retain vars;
+  if first.indxname then do;
+    idxcnt+1;
+    nom='';
+    uni='';
+    vars=name;
+  end;
+  else vars=catx(' ',vars,name);
+  if last.indxname then do;
+    if nomiss='yes' then nom='/nomiss';
+    if unique='yes' then uni='/unique';
+    call symputx('indexes'
+      ,catx(' ',symget('indexes'),indxname,'=(',vars,')',nom,uni)
+      ,'l');
+  end;
+  if last then call symputx('indexes',cats(symget('indexes'),'))'),'l');
+run;
+
+
+data;run;
+%let setds=&syslast;
+proc sql
+%if %datatyp(&maxobs)=NUMERIC %then %do;
+  outobs=&maxobs;
+%end;
+  ;
+  create table &setds as select * from &base_ds
+%if &random_sample=YES %then %do;
+  order by ranuni(42)
+%end;
+  ;
+reset outobs=max;
+create table datalines1 as
+  select name,type,length,varnum,format,label from dictionary.columns
+  where upcase(libname)="%upcase(%scan(&base_ds,1))"
+    and upcase(memname)="%upcase(%scan(&base_ds,2))";
+
+/**
+  Due to long decimals cannot use best. format
+  So - use bestd. format and then use character functions to strip trailing
+    zeros, if NOT an integer or missing!!  Cannot use int() as it upsets
+    note2err when there are missings.
+  resolved code = ifc( mod(coalesce(VARIABLE,0),1)=0
+    ,put(VARIABLE,best32.)
+    ,substrn(put(VARIABLE,bestd32.),1
+    ,findc(put(VARIABLE,bestd32.),'0','TBK')));
+**/
+
+data datalines_2;
+  format dataline $32000.;
+  set datalines1 (where=(upcase(name) not in
+    ('PROCESSED_DTTM','VALID_FROM_DTTM','VALID_TO_DTTM')));
+  if type='num' then dataline=
+    cats('ifc(mod(coalesce(',name,',0),1)=0
+      ,put(',name,',best32.-l)
+      ,substrn(put(',name,',bestd32.-l),1
+      ,findc(put(',name,',bestd32.-l),"0","TBK")))');
+  /**
+    * binary data must be converted, to store in text format.  It is identified
+    * by the presence of the $HEX keyword in the format.
+    */
+  else if upcase(format)=:'$HEX' then
+    dataline=cats('put(trim(',name,'),',format,')');
+  /**
+    * There is no easy way to store line breaks in a cards file.
+    * To discuss this, use: https://github.com/sasjs/core/issues/80
+    * Removing all nonprintables with kw (keep writeable)
+    */
+  else dataline=cats('compress(',name,', ,"kw")');
+run;
+
+proc sql noprint;
+select dataline into: datalines separated by ',' from datalines_2;
+
+%local
+  process_dttm_flg
+  valid_from_dttm_flg
+  valid_to_dttm_flg
+;
+%let process_dttm_flg = N;
+%let valid_from_dttm_flg = N;
+%let valid_to_dttm_flg = N;
+data _null_;
+  set datalines1 ;
+/* build attrib statement */
+  if type='char' then type2='$';
+  if strip(format) ne '' then format2=cats('format=',format);
+  if strip(label) ne '' then label2=cats('label=',quote(trim(label)));
+  str1=catx(' ',(put(name,$33.)||'length=')
+        ,put(cats(type2,length),$7.)||format2,label2);
+
+
+/* Build input statement */
+  if upcase(format)=:'$HEX' then type3=':'!!format;
+  else if type='char' then type3=':$char.';
+  str2=put(name,$33.)||type3;
+
+
+  if(upcase(name) = "PROCESSED_DTTM") then
+    call symputx("process_dttm_flg", "Y", "L");
+  if(upcase(name) = "VALID_FROM_DTTM") then
+    call symputx("valid_from_dttm_flg", "Y", "L");
+  if(upcase(name) = "VALID_TO_DTTM") then
+    call symputx("valid_to_dttm_flg", "Y", "L");
+
+
+  call symputx(cats("attrib_stmt_", put(_N_, 8.)), str1, "L");
+  call symputx(cats("input_stmt_", put(_N_, 8.))
+    , ifc(upcase(name) not in
+      ('PROCESSED_DTTM','VALID_FROM_DTTM','VALID_TO_DTTM'), str2, ""), "L");
+run;
+
+data _null_;
+  file &cards_file. &outencoding lrecl=32767 termstr=nl &append;
+  length __attrib $32767;
+  if _n_=1 then do;
+    put '/**';
+    put '  @file';
+    put "  @brief Datalines for %upcase(%scan(&base_ds,2)) dataset";
+    put "  @details Generated by %nrstr(%%)mp_ds2cards()";
+    put "    Source: https://github.com/sasjs/core";
+    put '  @cond ';
+    put '**/';
+    put "data &tgt_ds &indexes;";
+    put "attrib ";
+    %do i = 1 %to &nvars;
+      __attrib=symget("attrib_stmt_&i");
+      put __attrib;
+    %end;
+    put ";";
+
+    %if &process_dttm_flg. eq Y %then %do;
+      put 'retain PROCESSED_DTTM %sysfunc(datetime());';
+    %end;
+    %if &valid_from_dttm_flg. eq Y %then %do;
+      put 'retain VALID_FROM_DTTM &low_date;';
+    %end;
+    %if &valid_to_dttm_flg. eq Y %then %do;
+      put 'retain VALID_TO_DTTM &high_date;';
+    %end;
+    if __nobs=0 then do;
+      put 'call missing(of _all_);/* avoid uninitialised notes */';
+      put 'stop;';
+      put 'run;';
+    end;
+    else do;
+      put "infile cards dsd;";
+      put "input ";
+      %do i = 1 %to &nvars.;
+        %if(%length(&&input_stmt_&i..)) %then
+          put "  &&input_stmt_&i..";
+        ;
+      %end;
+      put ";";
+      put 'missing a b c d e f g h i j k l m n o p q r s t u v w x y z _;';
+      put "datalines4;";
+    end;
+  end;
+  set &setds end=__lastobs nobs=__nobs;
+/* remove all formats for write purposes - some have long underlying decimals */
+  format _numeric_ best30.29;
+  length __dataline $32767;
+  __dataline=catq('cqtmb',&datalines);
+  put __dataline;
+  if __lastobs then do;
+    put ';;;;';
+    put 'run;';
+    put '/** @endcond **/';
+    stop;
+  end;
+run;
+proc sql;
+  drop table &setds;
+quit;
+
+%if &showlog=YES %then %do;
+  data _null_;
+    infile &cards_file lrecl=32767;
+    input;
+    put _infile_;
+  run;
+%end;
+
+%put NOTE: CARDS FILE SAVED IN:;
+%put NOTE-;%put NOTE-;
+%put NOTE- %sysfunc(dequote(&cards_file.));
+%put NOTE-;%put NOTE-;
+%mend mp_ds2cards;
+/** @endcond **/

package/macros/mp_ds2csv.sas

@@ -0,0 +1,212 @@
+/**
+  @file
+  @brief Export a dataset to a CSV file WITH leading blanks
+  @details Export a dataset to a file or fileref, retaining leading blanks.
+
+  When using SASJS headerformat, the input statement is provided in the first
+  row of the CSV.
+
+  Usage:
+
+      %mp_ds2csv(sashelp.class,outref="%sysfunc(pathname(work))/file.csv")
+
+      filename example temp;
+      %mp_ds2csv(sashelp.air,outref=example,headerformat=SASJS)
+      data; infile example; input;put _infile_; if _n_>5 then stop;run;
+
+      data _null_;
+        infile example;
+        input;
+        call symputx('stmnt',_infile_);
+        stop;
+      run;
+      data work.want;
+        infile example dsd firstobs=2;
+        input &stmnt;
+      run;
+
+  Why use mp_ds2csv over, say, proc export?
+
+  1. Ability to retain leading blanks (this is a major one)
+  2. Control the header format
+  3. Simple one-liner
+
+  @param [in] ds The dataset to be exported
+  @param [in] dlm= (COMMA) The delimeter to apply.  For SASJS, will always be
+    COMMA. Supported values:
+    @li COMMA
+    @li SEMICOLON
+  @param [in] headerformat= (LABEL) The format to use for the header section.
+    Valid values:
+    @li LABEL - Use the variable label (or name, if blank)
+    @li NAME - Use the variable name
+    @li SASJS - Used to create sasjs-formatted input CSVs, eg for use in
+      mp_testservice.sas.  This format will supply an input statement in the
+      first row, making ingestion by datastep a breeze.  Special misisng values
+      will be prefixed with a period (eg `.A`) to enable ingestion on both SAS 9
+      and Viya.  Dates / Datetimes etc are identified by the format type (lookup
+      with mcf_getfmttype.sas) and converted to human readable formats (not
+      numbers).
+  @param [out] outfile= The output filename - should be quoted.
+  @param [out] outref= (0) The output fileref (takes precedence if provided)
+  @param [in] outencoding= (0) The (quoted) output encoding to use, eg `"UTF-8"`
+  @param [in] termstr= (CRLF) The line seperator to use.  For SASJS, will
+    always be CRLF.  Valid values:
+    @li CRLF
+    @li LF
+
+  <h4> SAS Macros </h4>
+  @li mcf_getfmttype.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarformat.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+
+  @version 9.2
+  @author Allan Bowe (credit mjsq)
+**/
+
+%macro mp_ds2csv(ds
+  ,dlm=COMMA
+  ,outref=0
+  ,outfile=
+  ,outencoding=0
+  ,headerformat=LABEL
+  ,termstr=CRLF
+)/*/STORE SOURCE*/;
+
+%local outloc delim i varlist var vcnt vat dsv vcom vmiss fmttype vfmt;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WARN)ING:  &ds does not exist;
+  %return;
+%end;
+
+%if %index(&ds,.)=0 %then %let ds=WORK.&ds;
+
+%if &outencoding=0 %then %let outencoding=;
+%else %let outencoding=encoding=&outencoding;
+
+%if &outref=0 %then %let outloc=&outfile;
+%else %let outloc=&outref;
+
+%if &headerformat=SASJS %then %do;
+  %let delim=",";
+  %let termstr=CRLF;
+  %mcf_getfmttype(wrap=YES)
+%end;
+%else %if &dlm=COMMA %then %let delim=",";
+%else %let delim=";";
+
+/* credit to mjsq - https://stackoverflow.com/a/55642267 */
+
+/* first get headers */
+data _null_;
+  file &outloc &outencoding lrecl=32767 termstr=&termstr;
+  length header $ 2000 varnm vfmt $32 dlm $1 fmttype $8;
+  call missing(of _all_);
+  dsid=open("&ds.","i");
+  num=attrn(dsid,"nvars");
+  dlm=&delim;
+  do i=1 to num;
+    varnm=upcase(varname(dsid,i));
+    if i=num then dlm='';
+  %if &headerformat=NAME %then %do;
+    header=cats(varnm,dlm);
+  %end;
+  %else %if &headerformat=LABEL %then %do;
+    header = cats(coalescec(varlabel(dsid,i),varnm),dlm);
+  %end;
+  %else %if &headerformat=SASJS %then %do;
+    if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.');
+    else do;
+      vfmt=coalescec(varfmt(dsid,i),'0');
+      fmttype=mcf_getfmttype(vfmt);
+      if fmttype='DATE' then header=cats(varnm,':date9.');
+      else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6');
+      else if fmttype='TIME' then header=cats(varnm,':TIME12.');
+      else header=cats(varnm,':best.');
+    end;
+  %end;
+  %else %do;
+    %put &sysmacroname: Invalid headerformat value (&headerformat);
+    %return;
+  %end;
+    put header @;
+  end;
+  rc=close(dsid);
+run;
+
+%let varlist=%mf_getvarlist(&ds);
+%let vcnt=%sysfunc(countw(&varlist));
+
+/**
+  * The $quote modifier (without a width) will take the length from the variable
+  * and increase by two.  However this will lead to truncation where the value
+  * contains double quotes (which are doubled up).  To get around this, scan the
+  * data to see the max number of double quotes, so that the appropriate width
+  * can be applied in the subsequent step.
+  */
+data _null_;
+  set &ds end=last;
+%do i=1 %to &vcnt;
+  %let var=%scan(&varlist,&i);
+  %if %mf_getvartype(&ds,&var)=C %then %do;
+    %let dsv1=%mf_getuniquename(prefix=csvcol1_);
+    %let dsv2=%mf_getuniquename(prefix=csvcol2_);
+    retain &dsv1 0;
+    &dsv2=length(&var)+countc(&var,'"');
+    if &dsv2>&dsv1 then &dsv1=&dsv2;
+    if last then call symputx(
+      "vlen&i"
+      /* should be no shorter than varlen, and no longer than 32767 */
+      ,cats('$quote',min(&dsv1+2,32767),'.')
+      ,'l'
+    );
+  %end;
+%end;
+
+%let vat=@;
+%let vcom=&delim;
+%let vmiss=%mf_getuniquename(prefix=csvcol3_);
+/* next, export data */
+data _null_;
+  set &ds.;
+  file &outloc mod dlm=&delim dsd &outencoding lrecl=32767 termstr=&termstr;
+  if _n_=1 then &vmiss='  ';
+  %do i=1 %to &vcnt;
+    %let var=%scan(&varlist,&i);
+    %if &i=&vcnt %then %do;
+      %let vat=;
+      %let vcom=;
+    %end;
+    %if %mf_getvartype(&ds,&var)=N %then %do;
+      %if &headerformat = SASJS %then %do;
+        %let vcom=&delim;
+        %let fmttype=%sysfunc(mcf_getfmttype(%mf_getvarformat(&ds,&var)0));
+        %if &fmttype=DATE %then %let vfmt=DATE9.;
+        %else %if &fmttype=DATETIME %then %let vfmt=E8601DT26.6;
+        %else %if &fmttype=TIME %then %let vfmt=TIME12.;
+        %else %do;
+          %let vfmt=;
+          %let vcom=;
+        %end;
+      %end;
+      %else %let vcom=;
+
+      /* must use period - in order to work in both 9.4 and Viya 3.5 */
+      if missing(&var) and &var ne %sysfunc(getoption(MISSING)) then do;
+        &vmiss=cats('.',&var);
+        put &vmiss &vat;
+      end;
+      else put &var &vfmt &vcom &vat;
+
+    %end;
+    %else %do;
+      %if &i ne &vcnt %then %let vcom=&delim;
+      put &var &&vlen&i &vcom &vat;
+    %end;
+  %end;
+run;
+
+%mend mp_ds2csv;

package/macros/mp_ds2ddl.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief A wrapper for mp_getddl.sas
+  @details In the next release, this will be the main version.
+
+
+  <h4> SAS Macros </h4>
+  @li mp_getddl.sas
+
+**/
+
+%macro mp_ds2ddl(libds,fref=getddl,flavour=SAS,showlog=YES,schema=
+  ,applydttm=NO
+)/*/STORE SOURCE*/;
+
+%local libref;
+%let libds=%upcase(&libds);
+%let libref=%scan(&libds,1,.);
+%if &libref=&libds %then %let libds=WORK.&libds;
+
+%mp_getddl(%scan(&libds,1,.)
+  ,%scan(&libds,2,.)
+  ,fref=&fref
+  ,flavour=SAS
+  ,showlog=&showlog
+  ,schema=&schema
+  ,applydttm=&applydttm
+)
+
+%mend mp_ds2ddl;

package/macros/mp_ds2fmtds.sas

@@ -0,0 +1,105 @@
+/**
+  @file
+  @brief Converts every value in a dataset to formatted value
+  @details Converts every value to it's formatted value.  All variables will
+  become character, and will be in the same order as the original dataset.
+
+  Lengths will be adjusted according to the format lengths, where applicable.
+
+  Usage:
+
+      %mp_ds2fmtds(sashelp.cars,work.cars)
+      %mp_ds2fmtds(sashelp.vallopt,vw_vallopt)
+
+  @param [in] libds The library.dataset to be converted
+  @param [out] outds The dataset to create.
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+
+  <h4> Related Macros </h4>
+  @li mp_jsonout.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_ds2fmtds(libds, outds
+)/*/STORE SOURCE*/;
+
+/* validations */
+
+%if not %mf_existds(libds=&libds) %then %do;
+  %put %str(WARN)ING:  &libds does not exist as either a VIEW or DATASET;
+  %return;
+%end;
+%if %index(&libds,.)=0 %then %let libds=WORK.&libds;
+
+/* grab metadata */
+proc contents noprint data=&libds
+  out=_data_(keep=name type length format formatl formatd varnum);
+run;
+proc sort;
+  by varnum;
+run;
+
+/* prepare formats and varnames */
+data _null_;
+  set &syslast end=last;
+  name=upcase(name);
+  /* fix formats */
+  if type=2 or type=6 then do;
+    length fmt $49.;
+    if format='' then fmt=cats('$',length,'.');
+    else if formatl=0 then fmt=cats(format,'.');
+    else fmt=cats(format,formatl,'.');
+    newlen=max(formatl,length);
+  end;
+  else do;
+    if format='' then fmt='best.';
+    else if formatl=0 then fmt=cats(format,'.');
+    else if formatd=0 then fmt=cats(format,formatl,'.');
+    else fmt=cats(format,formatl,'.',formatd);
+    /* needs to be wide, for datetimes etc */
+    newlen=max(length,formatl,24);
+  end;
+  /* 32 char unique name */
+  newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
+
+  call symputx(cats('name',_n_),name,'l');
+  call symputx(cats('newname',_n_),newname,'l');
+  call symputx(cats('len',_n_),newlen,'l');
+  call symputx(cats('fmt',_n_),fmt,'l');
+  call symputx(cats('type',_n_),type,'l');
+  if last then call symputx('nobs',_n_,'l');
+run;
+
+/* clean up */
+proc sql;
+drop table &syslast;
+
+%if &nobs=0 %then %do;
+  %put Dataset &libds has no columns!
+  data &outds;
+    set &libds;
+  run;
+  %return;
+%end;
+
+data &outds;
+  /* rename on entry */
+  set &libds(rename=(
+%local i;
+%do i=1 %to &nobs;
+  &&name&i=&&newname&i
+%end;
+  ));
+%do i=1 %to &nobs;
+  length &&name&i $&&len&i;
+  &&name&i=left(put(&&newname&i,&&fmt&i));
+  drop &&newname&i;
+%end;
+  if _error_ then call symputx('syscc',1012);
+run;
+
+%mend mp_ds2fmtds;