@sasjs/utils 2.48.2 → 2.49.1

package/macros/mp_csv2ds.sas

@@ -0,0 +1,140 @@
+/**
+  @file mp_csv2ds.sas
+  @brief Efficient import of arbitrary CSV using a dataset as template
+  @details Used to import relevant columns from a large CSV using
+  a dataset to provide the types and lengths.  Assumes that a header
+  row is provided, and datarows start on line 2.  Extra columns in
+  both the CSV and base dataset are ignored.
+
+  Usage:
+
+      filename mycsv temp;
+      data _null_;
+        file mycsv;
+        put 'name,age,nickname';
+        put 'John,48,Jonny';
+        put 'Jennifer,23,Jen';
+      run;
+
+      %mp_csv2ds(inref=mycsv,outds=myds,baseds=sashelp.class)
+
+
+  @param inref= fileref to the CSV
+  @param outds= output ds (lib.ds format)
+  @param view= Set to YES or NO to determine whether the output should be
+    a view or not.  Default is NO (not a view).
+  @param baseds= Template dataset on which to create the input statement.
+    Is used to determine types, lengths, and any informats.
+
+  @version 9.2
+  @author Allan Bowe
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mf_existds.sas
+
+**/
+
+%macro mp_csv2ds(inref=0,outds=0,baseds=0,view=NO);
+
+%mp_abort(iftrue=( &inref=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the INREF variable must be provided)
+)
+%mp_abort(iftrue=( %superq(outds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the OUTDS variable must be provided)
+)
+%mp_abort(iftrue=( &baseds=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS variable must be provided)
+)
+%mp_abort(iftrue=( %mf_existds(&baseds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS dataset (&baseds) needs to be assigned, and to exist)
+)
+
+/* count rows */
+%local hasheader; %let hasheader=0;
+data _null_;
+  if _N_ > 1 then do;
+    call symputx('hasheader',1,'l');
+    stop;
+  end;
+  infile &inref;
+  input;
+run;
+%mp_abort(iftrue=( &hasheader=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(No header row in &inref)
+)
+
+/* get the variables in the CSV */
+data _data_;
+  infile &inref;
+  input;
+  length name $32;
+  do i=1 to countc(_infile_,',')+1;
+    name=upcase(scan(_infile_,i,','));
+    output;
+  end;
+  stop;
+run;
+%local csv_vars;%let csv_vars=&syslast;
+
+/* get the variables in the dataset */
+proc contents noprint data=&baseds
+  out=_data_ (keep=name type length format: informat);
+run;
+%local base_vars; %let base_vars=&syslast;
+
+proc sql undo_policy=none;
+create table &csv_vars as
+  select a.*
+    ,b.type
+    ,b.length
+    ,b.format
+    ,b.formatd
+    ,b.formatl
+    ,b.informat
+  from &csv_vars a
+  left join &base_vars b
+  on a.name=upcase(b.name)
+  order by i;
+
+/* prepare the input statement */
+%local instat dropvars;
+data _null_;
+  set &syslast end=last;
+  length in dropvars $32767;
+  retain in dropvars;
+  if missing(type) then do;
+    informat='$1.';
+    dropvars=catx(' ',dropvars,name);
+  end;
+  else if missing(informat) then do;
+    if type=1 then informat='best.';
+    else informat=cats('$',length,'.');
+  end;
+  else informat=cats(informat,'.');
+  in=catx(' ',in,name,':',informat);
+  if last then do;
+    call symputx('instat',in,'l');
+    call symputx('dropvars',dropvars,'l');
+  end;
+run;
+
+/* import the CSV */
+data &outds
+  %if %upcase(&view)=YES %then %do;
+    /view=&outds
+  %end;
+  ;
+  infile &inref dsd firstobs=2;
+  input &instat;
+  %if %length(&dropvars)>0 %then %do;
+    drop &dropvars;
+  %end;
+run;
+
+%mend mp_csv2ds;

package/macros/mp_deleteconstraints.sas

@@ -0,0 +1,52 @@
+/**
+  @file mp_deleteconstraints.sas
+  @brief Delete constraionts
+  @details Takes the output from mp_getconstraints.sas as input
+
+        proc sql;
+        create table work.example(
+          TX_FROM float format=datetime19.,
+          DD_TYPE char(16),
+          DD_SOURCE char(2048),
+          DD_SHORTDESC char(256),
+          constraint pk primary key(tx_from, dd_type,dd_source),
+          constraint unq unique(tx_from, dd_type),
+          constraint nnn not null(DD_SHORTDESC)
+        );
+
+      %mp_getconstraints(lib=work,ds=example,outds=work.constraints)
+      %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
+
+  @param inds= The input table containing the constraint info
+  @param outds= a table containing the drop statements (drop_statement column)
+  @param execute= `YES|NO` - default is NO. To actually drop, use YES.
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_deleteconstraints(inds=mp_getconstraints
+  ,outds=mp_deleteconstraints
+  ,execute=NO
+)/*/STORE SOURCE*/;
+
+proc sort data=&inds out=&outds;
+  by libref table_name constraint_name;
+run;
+
+data &outds;
+  set &outds;
+  by libref table_name constraint_name;
+  length drop_statement $500;
+  if _n_=1 and "&execute"="YES" then call execute('proc sql;');
+  if first.constraint_name then do;
+    drop_statement=catx(" ","alter table",libref,".",table_name
+      ,"drop constraint",constraint_name,";");
+    output;
+    if "&execute"="YES" then call execute(drop_statement);
+  end;
+run;
+
+%mend mp_deleteconstraints;

package/macros/mp_deletefolder.sas

@@ -0,0 +1,70 @@
+/**
+  @file
+  @brief A macro to delete a directory
+  @details Will delete all folder content (including subfolder content) and
+    finally, the folder itself.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_deletefolder(&rootdir)
+
+  @param path Unquoted path to the folder to delete.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_deletefolder.test.sas
+
+**/
+
+%macro mp_deletefolder(folder);
+  /* proceed if valid directory */
+  %if %mf_isdir(&folder)=1 %then %do;
+
+    /* prep temp table */
+    %local tempds;
+    %let tempds=%mf_getuniquename();
+
+    /* recursive directory listing */
+    %mp_dirlist(path=&folder,outds=work.&tempds, maxdepth=MAX)
+
+    /* sort descending level so can delete folder contents before folders */
+    proc sort data=work.&tempds;
+      by descending level;
+    run;
+
+    /* ensure top level folder is removed at the end */
+    proc sql;
+    insert into work.&tempds set filepath="&folder";
+
+    /* delete everything */
+    data _null_;
+      set work.&tempds end=last;
+      length fref $8;
+      fref='';
+      rc=filename(fref,filepath);
+      rc=fdelete(fref);
+      if rc then do;
+        msg=sysmsg();
+        put "&sysmacroname:" / rc= / msg= / filepath=;
+      end;
+      rc=filename(fref);
+    run;
+
+    /* tidy up */
+    proc sql;
+    drop table work.&tempds;
+
+  %end;
+  %else %put &sysmacroname: &folder: is not a valid / accessible folder. ;
+%mend mp_deletefolder;

package/macros/mp_dictionary.sas

@@ -0,0 +1,52 @@
+/**
+  @file mp_dictionary.sas
+  @brief Creates a portal (libref) into the SQL Dictionary Views
+  @details Provide a libref and the macro will create a series of views against
+  each view in the special PROC SQL dictionary libref.
+
+  This is useful if you would like to visualise (navigate) the views in a SAS
+  client such as Base SAS, Enterprise Guide, or Studio (or [Data Controller](
+  https://datacontroller.io)).
+
+  It works by extracting the dictionary.dictionaries view into
+  YOURLIB.dictionaries, then uses that to create a YOURLIB.{viewName} for every
+  other dictionary.view, eg:
+
+      proc sql;
+      create view YOURLIB.columns as select * from dictionary.columns;
+
+  Usage:
+
+      libname demo "/lib/directory";
+      %mp_dictionary(lib=demo)
+
+  Or, to just create them in WORK:
+
+      %mp_dictionary()
+
+  If you'd just like to browse the dictionary data model, you can also check
+  out [this article](https://rawsas.com/dictionary-of-dictionaries/).
+
+  ![](https://user-images.githubusercontent.com/4420615/188278365-2987db97-0594-4a39-ac81-dbacdef5cdc8.png)
+
+  @param lib= (WORK) The libref in which to create the views
+
+  <h4> Related Files </h4>
+  @li mp_dictionary.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_dictionary(lib=WORK)/*/STORE SOURCE*/;
+  %local list i mem;
+  proc sql noprint;
+  create view &lib..dictionaries as select * from dictionary.dictionaries;
+  select distinct memname into: list separated by ' '  from &lib..dictionaries;
+  %do i=1 %to %sysfunc(countw(&list,%str( )));
+    %let mem=%scan(&list,&i,%str( ));
+    create view &lib..&mem as select * from dictionary.&mem;
+  %end;
+  quit;
+%mend mp_dictionary;

package/macros/mp_dirlist.sas

@@ -0,0 +1,263 @@
+/**
+  @file
+  @brief Returns all files and subdirectories within a specified parent
+  @details When used with getattrs=NO, is not OS specific (uses dopen / dread).
+
+  Credit for the rename approach:
+  https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003
+
+  Usage:
+
+      %mp_dirlist(path=/some/location, outds=myTable, maxdepth=MAX)
+
+      %mp_dirlist(outds=cwdfileprops, getattrs=YES)
+
+      %mp_dirlist(fref=MYFREF)
+
+  @warning In a Unix environment, the existence of a named pipe will cause this
+  macro to hang.  Therefore this tool should be used with caution in a SAS 9 web
+  application, as it can use up all available multibridge sessions if requests
+  are resubmitted.
+  If anyone finds a way to positively identify a named pipe using SAS (without
+  X CMD) do please raise an issue!
+
+
+  @param [in] path= (%sysfunc(pathname(work))) Path for which to return contents
+  @param [in] fref= (0) Provide a DISK engine fileref as an alternative to PATH
+  @param [in] maxdepth= (0) Set to a positive integer to indicate the level of
+    subdirectory scan recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+  @param [in] showparent= (NO) By default, the initial parent directory is not
+    part of the results.  Set to YES to include it.  For this record only,
+    directory=filepath.
+  @param [out] outds= (work.mp_dirlist) The output dataset to create
+  @param [out] getattrs= (NO)  If getattrs=YES then the doptname / foptname
+    functions are used to scan all properties - any characters that are not
+    valid in a SAS name (v7) are simply stripped, and the table is transposed
+    so theat each property is a column and there is one file per row.  An
+    attempt is made to get all properties whether a file or folder, but some
+    files/folders cannot be accessed, and so not all properties can / will be
+    populated.
+
+
+  @returns outds contains the following variables:
+    - directory (containing folder)
+    - file_or_folder (file / folder)
+    - filepath (path/to/file.name)
+    - filename (just the file name)
+    - ext (.extension)
+    - msg (system message if any issues)
+    - level (depth of folder)
+    - OS SPECIFIC variables, if <code>getattrs=</code> is used.
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mf_getvarlist.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_dropmembers.sas
+
+  <h4> Related Macros </h4>
+  @li mp_dirlist.test.sas
+
+  @version 9.2
+**/
+
+%macro mp_dirlist(path=%sysfunc(pathname(work))
+    , fref=0
+    , outds=work.mp_dirlist
+    , getattrs=NO
+    , showparent=NO
+    , maxdepth=0
+    , level=0 /* The level of recursion to perform.  For internal use only. */
+)/*/STORE SOURCE*/;
+%let getattrs=%upcase(&getattrs)XX;
+
+/* temp table */
+%local out_ds;
+data;run;
+%let out_ds=%str(&syslast);
+
+/* drop main (top) table if it exists */
+%if &level=0 %then %do;
+  %mp_dropmembers(%scan(&outds,-1,.), libref=WORK)
+%end;
+
+data &out_ds(compress=no
+    keep=file_or_folder filepath filename ext msg directory level
+  );
+  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
+    ext $20 msg $200 foption $16;
+  if _n_=1 then call missing(of _all_);
+  retain level &level;
+  %if &fref=0 %then %do;
+    rc = filename(fref, "&path");
+  %end;
+  %else %do;
+    fref="&fref";
+    rc=0;
+  %end;
+  if rc = 0 then do;
+    did = dopen(fref);
+    if did=0 then do;
+      putlog "NOTE: This directory is empty, or does not exist - &path";
+      msg=sysmsg();
+      put msg;
+      put _all_;
+      stop;
+    end;
+    /* attribute is OS-dependent - could be "Directory" or "Directory Name" */
+    numopts=doptnum(did);
+    do i=1 to numopts;
+      foption=doptname(did,i);
+      if foption=:'Directory' then i=numopts;
+    end;
+    directory=dinfo(did,foption);
+    rc = filename(fref);
+  end;
+  else do;
+    msg=sysmsg();
+    put _all_;
+    stop;
+  end;
+  dnum = dnum(did);
+  do i = 1 to dnum;
+    filename = dread(did, i);
+    filepath=cats(directory,'/',filename);
+    rc = filename(fref2,filepath);
+    midd=dopen(fref2);
+    dmsg=sysmsg();
+    if did > 0 then file_or_folder='folder';
+    rc=dclose(midd);
+    midf=fopen(fref2);
+    fmsg=sysmsg();
+    if midf > 0 then file_or_folder='file';
+    rc=fclose(midf);
+
+    if index(fmsg,'File is in use') or index(dmsg,'is not a directory')
+      then file_or_folder='file';
+    else if index(fmsg,'Insufficient authorization') then file_or_folder='file';
+    else if file_or_folder='' then file_or_folder='locked';
+
+    if file_or_folder='file' then do;
+      ext = prxchange('s/.*\.{1,1}(.*)/$1/', 1, filename);
+      if filename = ext then ext = ' ';
+    end;
+    else do;
+      ext='';
+      file_or_folder='folder';
+    end;
+    output;
+  end;
+  rc = dclose(did);
+  %if &showparent=YES and &level=0 %then %do;
+    filepath=directory;
+    file_or_folder='folder';
+    ext='';
+    filename=scan(directory,-1,'/\');
+    msg='';
+    level=&level;
+    output;
+  %end;
+  stop;
+run;
+
+%if %substr(&getattrs,1,1)=Y %then %do;
+  data &out_ds;
+    set &out_ds;
+    length infoname infoval $60 fref $8;
+    if _n_=1 then call missing(fref);
+    rc=filename(fref,filepath);
+    drop rc infoname fid i close fref;
+    if file_or_folder='file' then do;
+      fid=fopen(fref);
+      if fid le 0 then do;
+        msg=sysmsg();
+        putlog "Could not open file:" filepath fid= ;
+        sasname='_MCNOTVALID_';
+        output;
+      end;
+      else do i=1 to foptnum(fid);
+        infoname=foptname(fid,i);
+        infoval=finfo(fid,infoname);
+        sasname=compress(infoname, '_', 'adik');
+        if anydigit(sasname)=1 then sasname=substr(sasname,anyalpha(sasname));
+        if upcase(sasname) ne 'FILENAME' then output;
+      end;
+      close=fclose(fid);
+    end;
+    else do;
+      fid=dopen(fref);
+      if fid le 0 then do;
+        msg=sysmsg();
+        putlog "Could not open folder:" filepath fid= ;
+        sasname='_MCNOTVALID_';
+        output;
+      end;
+      else do i=1 to doptnum(fid);
+        infoname=doptname(fid,i);
+        infoval=dinfo(fid,infoname);
+        sasname=compress(infoname, '_', 'adik');
+        if anydigit(sasname)=1 then sasname=substr(sasname,anyalpha(sasname));
+        if upcase(sasname) ne 'FILENAME' then output;
+      end;
+      close=dclose(fid);
+    end;
+  run;
+  proc sort;
+    by filepath sasname;
+  proc transpose data=&out_ds out=&out_ds(drop=_:);
+    id sasname;
+    var infoval;
+    by filepath file_or_folder filename ext ;
+  run;
+%end;
+
+data &out_ds;
+  set &out_ds(where=(filepath ne ''));
+run;
+
+/**
+  * The above transpose can mean that some updates create additional columns.
+  * This necessitates the occasional use of datastep over proc append.
+  */
+%if %mf_existds(&outds) %then %do;
+  %local basevars appvars newvars;
+  %let basevars=%mf_getvarlist(&outds);
+  %let appvars=%mf_getvarlist(&out_ds);
+  %let newvars=%length(%mf_wordsinstr1butnotstr2(Str1=&appvars,Str2=&basevars));
+  %if &newvars>0 %then %do;
+    data &outds;
+      set &outds &out_ds;
+    run;
+  %end;
+  %else %do;
+    proc append base=&outds data=&out_ds force nowarn;
+    run;
+  %end;
+%end;
+%else %do;
+  proc append base=&outds data=&out_ds;
+  run;
+%end;
+
+/* recursive call */
+%if &maxdepth>&level or &maxdepth=MAX %then %do;
+  data _null_;
+    set &out_ds;
+    where file_or_folder='folder';
+  %if &showparent=YES and &level=0 %then %do;
+    if filepath ne directory;
+  %end;
+    length code $10000;
+    code=cats('%nrstr(%mp_dirlist(path=',filepath,",outds=&outds"
+      ,",getattrs=&getattrs,level=%eval(&level+1),maxdepth=&maxdepth))");
+    put code=;
+    call execute(code);
+  run;
+%end;
+
+/* tidy up */
+proc sql;
+drop table &out_ds;
+
+%mend mp_dirlist;

package/macros/mp_distinctfmtvalues.sas

@@ -0,0 +1,50 @@
+/**
+  @file
+  @brief Creates a dataset containing distinct _formatted_ values
+  @details If no format is supplied, then the original value is used instead.
+    There is also a dependency on other macros within the Macro Core library.
+    Usage:
+
+        %mp_distinctfmtvalues(libds=sashelp.class,var=age,outvar=age,outds=test)
+
+  @param libds input dataset
+  @param var variable to get distinct values for
+  @param outvar variable to create.  Default:  `formatted_value`
+  @param outds dataset to create.  Default:  work.mp_distinctfmtvalues
+  @param varlen length of variable to create (default 200)
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_distinctfmtvalues(
+    libds=
+    ,var=
+    ,outvar=formatted_value
+    ,outds=work.mp_distinctfmtvalues
+    ,varlen=2000
+)/*/STORE SOURCE*/;
+
+  %local fmt vtype;
+  %let fmt=%mf_getvarformat(&libds,&var);
+  %let vtype=%mf_getvartype(&libds,&var);
+
+  proc sql;
+  create table &outds as
+    select distinct
+    %if &vtype=C & %trim(&fmt)=%str() %then %do;
+      &var
+    %end;
+    %else %if &vtype=C %then %do;
+      put(&var,&fmt)
+    %end;
+    %else %if %trim(&fmt)=%str() %then %do;
+        put(&var,32.)
+    %end;
+    %else %do;
+      put(&var,&fmt)
+    %end;
+        as &outvar length=&varlen
+    from &libds;
+%mend mp_distinctfmtvalues;

package/macros/mp_dropmembers.sas

@@ -0,0 +1,46 @@
+/**
+  @file
+  @brief Drops tables / views (if they exist) without warnings in the log
+  @details Useful for dropping tables when you're not sure they exist, or if
+  you are not sure whether they are a dataset or view.  Also efficient for
+  dropping multiple tables / views.
+
+  Example usage:
+
+      proc sql;
+      create table data1 as select * from sashelp.class;
+      create view view2 as select * from sashelp.class;
+      %mp_dropmembers(data1 view2, libref=WORK)
+
+
+  <h4> SAS Macros </h4>
+  @li mf_isblank.sas
+
+
+  @param [in] list space separated list of datasets / views, WITHOUT libref
+  @param [in] libref= (WORK) Note - you can only drop from one library at a time
+  @param [in] iftrue= (1=1) Conditionally drop tables, eg if &debug=N
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_dropmembers(
+    list /* space separated list of datasets / views */
+    ,libref=WORK  /* can only drop from a single library at a time */
+    ,iftrue=%str(1=1)
+)/*/STORE SOURCE*/;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
+  %if %mf_isblank(&list) %then %do;
+    %put NOTE: nothing to drop!;
+    %return;
+  %end;
+
+  proc datasets lib=&libref nolist;
+    delete &list;
+    delete &list /mtype=view;
+  run;
+%mend mp_dropmembers;